{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: Graphics Device\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "os.environ['THEANO_FLAGS']='device=gpu0'\n",
    "\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "np.random.seed(123)\n",
    "import matplotlib.pyplot as plt\n",
    "import lasagne\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "conv = lasagne.layers.Conv2DLayer\n",
    "pool = lasagne.layers.MaxPool2DLayer\n",
    "NUM_EPOCHS = 500\n",
    "BATCH_SIZE = 256\n",
    "LEARNING_RATE = 0.001\n",
    "DIM = 60\n",
    "NUM_CLASSES = 10\n",
    "mnist_cluttered = \"mnist_cluttered_60x60_6distortions.npz\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Spatial Transformer Network\n",
    "We use lasagne to classify cluttered MNIST digits using the spatial transformer network introduced in [1]. The spatial Transformer Network applies a learned affine transformation to its input.\n",
    "\n",
    "\n",
    "\n",
    "## Load data\n",
    "We test the spatial transformer network using cluttered MNIST data.\n",
    "\n",
    "**Download the data (41 mb) with:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2015-08-19 14:52:08--  https://s3.amazonaws.com/lasagne/recipes/datasets/mnist_cluttered_60x60_6distortions.npz\n",
      "Resolving s3.amazonaws.com... 54.231.48.99\n",
      "Connecting to s3.amazonaws.com|54.231.48.99|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 43046126 (41M) [application/octet-stream]\n",
      "Server file no newer than local file 'mnist_cluttered_60x60_6distortions.npz' -- not retrieving.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -N https://s3.amazonaws.com/lasagne/recipes/datasets/mnist_cluttered_60x60_6distortions.npz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples: (50000, 1, 60, 60)\n",
      "Validation samples: (10000, 1, 60, 60)\n",
      "Test samples: (10000, 1, 60, 60)\n"
     ]
    }
   ],
   "source": [
    "def load_data():\n",
    "    data = np.load(mnist_cluttered)\n",
    "    X_train, y_train = data['x_train'], np.argmax(data['y_train'], axis=-1)\n",
    "    X_valid, y_valid = data['x_valid'], np.argmax(data['y_valid'], axis=-1)\n",
    "    X_test, y_test = data['x_test'], np.argmax(data['y_test'], axis=-1)\n",
    "\n",
    "    # reshape for convolutions\n",
    "    X_train = X_train.reshape((X_train.shape[0], 1, DIM, DIM))\n",
    "    X_valid = X_valid.reshape((X_valid.shape[0], 1, DIM, DIM))\n",
    "    X_test = X_test.reshape((X_test.shape[0], 1, DIM, DIM))\n",
    "    \n",
    "    print \"Train samples:\", X_train.shape\n",
    "    print \"Validation samples:\", X_valid.shape\n",
    "    print \"Test samples:\", X_test.shape\n",
    "\n",
    "    return dict(\n",
    "        X_train=lasagne.utils.floatX(X_train),\n",
    "        y_train=y_train.astype('int32'),\n",
    "        X_valid=lasagne.utils.floatX(X_valid),\n",
    "        y_valid=y_valid.astype('int32'),\n",
    "        X_test=lasagne.utils.floatX(X_test),\n",
    "        y_test=y_test.astype('int32'),\n",
    "        num_examples_train=X_train.shape[0],\n",
    "        num_examples_valid=X_valid.shape[0],\n",
    "        num_examples_test=X_test.shape[0],\n",
    "        input_height=X_train.shape[2],\n",
    "        input_width=X_train.shape[3],\n",
    "        output_dim=10,)\n",
    "data = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAG2CAYAAAAwUvKlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFfhJREFUeJzt3X+w3WV9J/DPB5I7MEQwWwwELQYtsYhjGcEBt0vILkag\nIC501TpOqusgbtdhRNfFobOVdl1gRq21I4OVWl0ZmU4LOhva8mNLKw3xx24U7KyV1rKaQBeaFBGo\ngTVInv3je1LvXr7fk3tu7s39JHm9Zu6c3Of5Ps/3OeeenPd5vuf5fk+21gIAFtshiz0AAIgQSAAU\nIZAAKEEgAVCCQAKgBIEEQAkCiQWTmasyc1dmfnaxx1JZZq4dPU5XLfZYYDEJJCaSmT+bmZ/IzG9l\n5hOZ+aPM/D+Z+ceZ+Y7MnOppttcnu2Xmlsz83pj6XZn5pb3dzyKb1eM0LcB2ZeZ3x2y3LDOfnLbt\n8TPqt4zKn8zMFQN93D3a5iUDbY/vafPGzLwjM7dn5s7MfDQz/yozP5+ZvzzaZtW0cc32Z81sHh/2\nX0sWewDsPzLzgxFxVURkRHwlIu6KiH+MiGMjYk1EfDoifiUiXr1AQ9jTC/bBdpb3jyNiVWaua639\naU/9L0XEstF2h/bU7368lkXEb0T3t+sz9Lg+pzwzb4iISyLiqYj4k4j4XnTPl5Mi4oLonic3RsQP\nRvuc3kdG9/xqo7qZtg6MgwOEQGJWMvNXI+LXI+LBiHhja21zzzbnRMR/3MdDO5jdFRH/MiLeGRF9\ngfTOiHgkur/Z6T31Obp9ICIuyczfbq399ZjtxsrMfxFdGD0UEa9prT08o35JRJwVEdFaeyJ6Qmf3\nYcvW2n+ezT45sDhkxx5l5qrowmhnRPxCXxhFRLTW7oyIX5hFf3dn5q6BurePDs+8bfT72tG2x0c3\nG5h+COezmfm2aX2tnVF/1Yy+T8/MWzLz70eHGh/MzN/JzJVDY8zMpZn5wcz8m8z8v9M/D8vMF2Xm\ndZn53VHdo5m5ITNPG7hvx2Tm72Xmtsx8KjPv230Ia46+HxFfjIg3ZObRM/b1yuhmqp+JiGfH9NEi\n4sroZlAf3ouxRET889HtF2aGUUREa+3HrbU/28t9cAAzQ2I2/m10z5Xfb619e9yGrbWds+xztoff\nvhfdO+nLR7//1rRtvhndYZzfiO5Qz5aI+K/T6u/e/Y/MfEdE3BART0fErdG9i18d3Tv612fmGa21\nh3rG8cWIOC0ibhv9e/uov1dFxH+PiOURcUdE3BIRL4iIfx0RmzLzotba7dP2f3R0hzlPiIh7ImJT\nRBwXEb8T/bOb2WgR8bsR8ZaIeFtE/Oa0undGxK6I+L3oZlHj/LeI2BgRF2Tm2tba3XMcz6Oj29Vz\nbM/BrrXmx8/Yn4j4s+he3N4xYbtVo3afmVF+d0Q8O9Dm7aM2vzyjfEtEfHfMvnZFxJ8P1K2Obnb3\nnYhYOaPuX0X3GcsXe8a4K7rQ+2cz6pZEd5jrqYg4c0bdyoj4u4h4OCKmppXfMOrvN2dsf+pobLsi\n4oOzfFzXjra/cfT7dyLi/mn1h0f3Gc2do983jbY/vucxfTa6IyWnjf69eeBxeMlA2+OnlR032u+u\niNgQXVD+TETkBM+ZXUPPDT8H/o9DdszG7kNaf7eoo5i7X4kuRN7TWntkekVr7c8j4o+imyUd0dP2\n11prj80oOz8iXhIRn2it3TOjv0ci4iPRLfQ4OyIiM5dGxFsj4snoDn1O3/4bEXHT3O7WP/l0RLws\nM88c/f5vIuKo6GZPs9Ja+3pE/EFEnJqZb53LIFp3mO6iiPjfEfH66O7XdyLiicy8PTPfmplecxjk\nkB0Hg9eMbtdmZt+H+yui+wzlZRFx77TyFhH/c0x/qzLz13vqTxzdnhQRt0fEz0Y3a9ncWvvHnu3/\nIrpDbnP1uYj4UHSH6e6JiEsj4h+iOxQ3iSujC5SrM/OW1tqPJh1Ia+3uzFwdET8f3QKGU0b/Pmf0\n87bMvKDN/tAuBxGBxGw8Et2L6osWeyBz9FOj23ErAFtEPGeG1FrbNqa/N86yv6NGt319RUT8/Zh+\n9qi1ti0z/ygifjEzPxldAHy0tfbjCfvZmpmfiIj3R8R7Yo6LHFprLbrDhJt2l2XmuuiC87XRzVh/\ney59c2AzfWY2dh+WOnue+tsVETFw+Ob587SP6Z6ILiCObK0dMvBz6MzDb3voLyLiwj3096EZ2x8z\n0N+xc7xf090Q3SzsD+Mnix3m4uqIeCwirszMn9rTxrPVuvOk/tPo1z0tsuAgJZCYjc9GxDPRvQM/\nadyGA1dqmOkH0Z3b8pyz/KP7cL3Ps9F/cudubUz9V0f7m68z/b86up1tf/dHt7rvlMw8sqd+7TyM\n6U+jW3H4wojY2Fr727l00rrzgz4U3axuvi9l9MPR7azOa+LgI5DYo9ba1ug+jJ+KiD/JzFP7tsvM\n86JbAr0n/2N0+84Z7c+ObmVWn+9HxIrMPGxM/U8P1F0XXaD+VmaeOLMyM6emLQiYjQ3RfXD/7tF9\nfo7MfE1mHh7RnX8TEZ+PiCNjxqKG0TlLc1pEMN3oMNnF0S07v7Rvkwm6uz66+/euiHjxbBtl5rmZ\nefHoBNiZdcviJ0v3N04wFg4iPkNiVlpr145eaK6KiM2Z+ZWI+EZ073qPiW628DMR0XvS7Ayfje7z\nnCsz8+eim0GsjohzozvX5xd72twV3ezpjsy8JyJ+FBHfbK398bT6X8rMWyPivugC6C9aa/e01v5m\ndB7SZyLirzLzjoj424hYGt0s7czoPt95+Yx99r6Tb639ODMvjog7owvor0TEX0a3DPynozsh9YTo\nDsU9PWr2q9Ed8rx8FEJfjm714puiu8TOhXt81PagtXbf6L73GZqVPKe8tfZMZl4Z3eG/cYE0s+3L\nojtP7Aejv9ED0S2pf1F0KxOPioivRfcGAZ5DIDFrrbUPZebNEfHvo/sc4O0RcVh0J0R+MyKujW4m\nsKd+/iEzz4puefSa6FZjbY7uA++XRPdOf6b/Et3nS6+P7kP7Q6L7kHx3IL0nulnA2dFdLSKjO2H2\nntE+b8rMv4yI/zAa++uiC9OHo3vh/YOZw4wxs4rW2v8ahen7ortG29uj+2zs4eiC+teim7Xt3v77\nmfnzEXHN6D6cFhF/HRH/LrpDbXsdSGMM3ZfB+9hauyUzvxrdisLZtv18dEvb10XEz0X3t10W3SHa\neyPi5oj49KSLLTh4ZDfTB4DF5TMkAEoQSACUIJAAKGFBFzVk5jcWsn8A9i+ttd7TRiIWeFFDZlox\nAcA/aa0NnhjtkB0AJQgkAEoQSACUIJAAKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgkAEoQSACUIJAA\nKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgkAEoQSACUIJAAKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgk\nAEoQSACUIJAAKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgkAEoQSACUIJAAKEEgAVCCQAKgBIEEQAkC\nCYASBBIAJQgkAEoQSACUIJAAKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgkAEoQSACUIJAAKEEgAVCC\nQAKgBIEEQAkCCYASBBIAJSxZ7AHAfDnjjDMG69asWdNb/uEPf3ihhgNMyAwJgBIEEgAlCCQAShBI\nAJQgkAAoIVtrC9d55sJ1fgA74ogjesuvuuqqwTa33nprb/mmTZvmZUxVXHTRRYN111xzzWDdS1/6\n0t7yqampvR4TMHuttRyqM0MCoASBBEAJAgmAEgQSACUIJABKEEgAlGDZ9yLJHFz5GNdee21v+fvf\n//7BNnfddVdv+bnnnjvZwIpYsWJFb/nKlSsH24x7TIc89thjg3UPPvjgxP0B41n2DUB5AgmAEgQS\nACUIJABKEEgAlOArzBfJcccdN1h3ySWX9JY/9dRTg20+9alPzdsYli1bNthm1apVE+/nxS9+cW/5\nxRdfPHGb1atXD7aZy4rRRx55ZLDusssu6y3fsGHDxPsB9swMCYASBBIAJQgkAEoQSACUIJAAKEEg\nAVCCi6susKELfl599dWDba644oqJ+oqI2LFjR2/5Aw88MNhm+fLlveWHHXbYYJsXvOAFveVzeR7N\n5WKoN91002DduDFs2bKlt/wLX/jCxP1961vfGmwDjOfiqgCUJ5AAKEEgAVCCQAKgBIEEQAlW2S2w\ntWvX9pbfcccdg22WLOm/5u3HPvaxwTannnpqb/m4i6EOXcB0nKeffrq3/Prrr5+4r/vvv3+w7vbb\nb+8t3759+8T7GWchn//Ac1llB0B5AgmAEgQSACUIJABKEEgAlCCQACihf30xExl3kdCzzz67t3xo\naXdExBve8Ibe8ttuu22ygRVnyTUwnRkSACUIJABKEEgAlCCQAChBIAFQglV2C+zVr351b/m4FWab\nN2+euA3A/s4MCYASBBIAJQgkAEoQSACUIJAAKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgkAEoQSACU\n4OKqBZ188sm95du3b98n+z/88MMH66699tre8ssvv3yhhgMcJMyQAChBIAFQgkACoASBBEAJAgmA\nEgQSACVka23hOs9cuM73ExdddFFv+c033zzYZseOHROVR0TM5e9444039pZ/4AMfmLivQw7x3gbY\ns9ZaDtV5FQGgBIEEQAkCCYASBBIAJQgkAEqwym6RvOIVrxisO++883rL3/e+9w22Gfo7HnHEEYNt\njjzyyMG6ST377LODdVu3bu0tv+WWWwbbfP3rX+8tv+222wbbjFuFCNRglR0A5QkkAEoQSACUIJAA\nKEEgAVCCQAKgBMu+9yOZg6slY2pqqrf8Ix/5yGCbyy67bK/HtNtNN900WPfyl7+8t/zEE08cbHPY\nYYf1lt93332Dbd773vcO1n35y18erAP2Hcu+AShPIAFQgkACoASBBEAJAgmAEpYs9gCYvXErIt/0\npjf1lr/73e8ebLNr166Jx/DEE0/0lq9fv37ivs4444zBuuuuu663/FWvetVgm7POOmuwzio7qM8M\nCYASBBIAJQgkAEoQSACUIJAAKEEgAVCCi6sWtGzZst7ycRdKvfDCC3vLjz322HkZ0247d+7sLT/8\n8MPndT/HHHNMb/nDDz882GbHjh2Dda973et6y7/2ta9NNjBgr7i4KgDlCSQAShBIAJQgkAAoQSAB\nUIKLqy6SpUuXDtZdc801veWXXnrpQg1n1saNez4df/zxE7f54Q9/OFj36KOP7s1wgH3ADAmAEgQS\nACUIJABKEEgAlCCQAChBIAFQgmXfCyyz/zqCn/vc5wbbvPnNb16o4SyYo446arBu/fr1veXnnHPO\nYJvzzjuvt3zbtm2DbdatWzdY98ADDwzW7QuXX375YN3HP/7xfTgSqMsMCYASBBIAJQgkAEoQSACU\nIJAAKMFXmE9gxYoVveXbt2+fuK+h1Xf70vLly3vLx13YdOirysetGjzhhBN6yx9//PHBNg899FBv\n+YYNGwbbfPvb3x6s++hHP9pbvpDP/+le+MIXDtbtqwvWQgW+whyA8gQSACUIJABKEEgAlCCQAChB\nIAFQgmXfM4xbjn3jjTf2lh999NET72fcEuU777yzt/zMM88cbHPaaadNPIaVK1f2lp988smDbeay\nXP3ee+/tLb/iiisG22zcuLG3fOfOnYNtxj2XDzmk/73Xvlr2Pc6SJa5xzMHDsm8AyhNIAJQgkAAo\nQSABUIJAAqAEq+wmcNlll/WWv/a1rx1sc/7550+8n6G/ydBKsXFt5tsNN9zQW/7JT35ysM3WrVt7\ny5988snBNkOr+Z555pkxoxs21J9VdrBvWWUHQHkCCYASBBIAJQgkAEoQSACUIJAAKMGy73kwNTU1\nWLd8+fJ528+4C5vuq+XL27dv3yf7t+wbDkyWfQNQnkACoASBBEAJAgmAEgQSACVY3jMPxn2t9rZt\n2/bhSA4cQ6vfli5dOq/7GbdycdOmTb3lp59++sT7Gfoad+AnzJAAKEEgAVCCQAKgBIEEQAkCCYAS\nBBIAJVj2zX5lvi+Gev755w/WnXLKKROPYahu3bp1kw0MDkJmSACUIJAAKEEgAVCCQAKgBIEEQAkC\nCYASLPvmoHbSSScN1k1NTU3c35YtW3rLH3/88Yn7goONGRIAJQgkAEoQSACUIJAAKEEgAVBCzvfF\nKv+/zjMXrnNYYJk5b30t5P8z2J+01gb/Y5khAVCCQAKgBIEEQAkCCYASBBIAJQgkAEpwcVUYYKk2\n7FtmSACUIJAAKEEgAVCCQAKgBIEEQAkCCYASBBIAJQgkAEoQSACUIJAAKEEgAVCCQAKgBBdXnQfP\nf/7zB+sOPfTQ3vITTzxxsM369et7y4899tjBNhdeeGFv+dKlSwfbAFRihgRACQIJgBIEEgAlCCQA\nShBIAJSQC/k1zZnpO6DnIDN7y6empgbbvPKVr+wt37x587yMCWA+tNb6X+DCDAmAIgQSACUIJABK\nEEgAlCCQAChBIAFQgmXfAOwzln0DUJ5AAqAEgQRACQIJgBIEEgAl+ArzA8SaNWt6yzdu3LiPRwIw\nN2ZIAJQgkAAoQSABUIJAAqAEgQRACQIJgBIs+96PvOUtbxmsu/7663vLly9fvlDDAZhXZkgAlCCQ\nAChBIAFQgkACoASBBEAJVtnNcO655+6T/bzrXe8arDvuuON6y1evXj3Y5nnPe95ejwlgMZkhAVCC\nQAKgBIEEQAkCCYASBBIAJQgkAErI1trCdZ65cJ3vpczsLX/mmWcWdf9zNfR3XLLEyn6gjtba4Iuf\nGRIAJQgkAEoQSACUIJAAKEEgAVCCJVgzzPfqt8cee6y3/P7775+4ry996UuDdRdccMHE/QFUYoYE\nQAkCCYASBBIAJQgkAEoQSACUIJAAKOGgvbjqkPle9r3YFvLvCzApF1cFoDyBBEAJAgmAEgQSACUI\nJABKcHHVGaxKA1gcZkgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQ\ngkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgA\nlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQS\nACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASB\nBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChB\nIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABK\nEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmA\nEgQSACUIJABKWLLA/d+7wP0DcIDI1tpijwEAHLIDoAaBBEAJAgmAEgQSACUIJABKEEgAlCCQAChB\nIAFQgkACoASBBEAJAgmAEgQSACUIJABKEEgAlCCQAChBIAFQgkACoASBBEAJAgmAEgQSACX8P2lI\niEbi9+h6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f39666ee510>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(7,7))\n",
    "plt.imshow(data['X_train'][101].reshape(DIM, DIM), cmap='gray', interpolation='none')\n",
    "plt.title('Cluttered MNIST', fontsize=20)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building the model\n",
    "We use a model where the localization network is a two layer convolution network which operates directly on the image input. The output from the localization network is a 6 dimensional vector specifying the parameters in the affine transformation. \n",
    "\n",
    "The localization feeds into the transformer layer which applies the transformation to the image input. In our setup the transformer layer downsamples the input by a factor 3. \n",
    "\n",
    "Finally a 2 layer convolution layer and 2 fully connected layers calculates the output probabilities. \n",
    "\n",
    "**The model**\n",
    "\n",
    "\n",
    "    Input -> localization_network -> TransformerLayer -> output_network -> predictions\n",
    "       |                                |\n",
    "       >--------------------------------^"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformer network output shape:  (None, 1, 20, 20)\n"
     ]
    }
   ],
   "source": [
    "def build_model(input_width, input_height, output_dim,\n",
    "                batch_size=BATCH_SIZE):\n",
    "    ini = lasagne.init.HeUniform()\n",
    "    l_in = lasagne.layers.InputLayer(shape=(None, 1, input_width, input_height),)\n",
    "\n",
    "    # Localization network\n",
    "    b = np.zeros((2, 3), dtype=theano.config.floatX)\n",
    "    b[0, 0] = 1\n",
    "    b[1, 1] = 1\n",
    "    b = b.flatten()\n",
    "    loc_l1 = pool(l_in, pool_size=(2, 2))\n",
    "    loc_l2 = conv(\n",
    "        loc_l1, num_filters=20, filter_size=(5, 5), W=ini)\n",
    "    loc_l3 = pool(loc_l2, pool_size=(2, 2))\n",
    "    loc_l4 = conv(loc_l3, num_filters=20, filter_size=(5, 5), W=ini)\n",
    "    loc_l5 = lasagne.layers.DenseLayer(\n",
    "        loc_l4, num_units=50, W=lasagne.init.HeUniform('relu'))\n",
    "    loc_out = lasagne.layers.DenseLayer(\n",
    "        loc_l5, num_units=6, b=b, W=lasagne.init.Constant(0.0), \n",
    "        nonlinearity=lasagne.nonlinearities.identity)\n",
    "    \n",
    "    # Transformer network\n",
    "    l_trans1 = lasagne.layers.TransformerLayer(l_in, loc_out, downsample_factor=3.0)\n",
    "    print \"Transformer network output shape: \", l_trans1.output_shape\n",
    "    \n",
    "    # Classification network\n",
    "    class_l1 = conv(\n",
    "        l_trans1,\n",
    "        num_filters=32,\n",
    "        filter_size=(3, 3),\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "    class_l2 = pool(class_l1, pool_size=(2, 2))\n",
    "    class_l3 = conv(\n",
    "        class_l2,\n",
    "        num_filters=32,\n",
    "        filter_size=(3, 3),\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "    class_l4 = pool(class_l3, pool_size=(2, 2))\n",
    "    class_l5 = lasagne.layers.DenseLayer(\n",
    "        class_l4,\n",
    "        num_units=256,\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "\n",
    "    l_out = lasagne.layers.DenseLayer(\n",
    "        class_l5,\n",
    "        num_units=output_dim,\n",
    "        nonlinearity=lasagne.nonlinearities.softmax,\n",
    "        W=ini,\n",
    "    )\n",
    "\n",
    "    return l_out, l_trans1\n",
    "\n",
    "model, l_transform = build_model(DIM, DIM, NUM_CLASSES)\n",
    "model_params = lasagne.layers.get_all_params(model, trainable=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = T.tensor4()\n",
    "y = T.ivector()\n",
    "\n",
    "# training output\n",
    "output_train = lasagne.layers.get_output(model, X, deterministic=False)\n",
    "\n",
    "# evaluation output. Also includes output of transform for plotting\n",
    "output_eval, transform_eval = lasagne.layers.get_output([model, l_transform], X, deterministic=True)\n",
    "\n",
    "sh_lr = theano.shared(lasagne.utils.floatX(LEARNING_RATE))\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(output_train, y))\n",
    "updates = lasagne.updates.adam(cost, model_params, learning_rate=sh_lr)\n",
    "\n",
    "train = theano.function([X, y], [cost, output_train], updates=updates)\n",
    "eval = theano.function([X], [output_eval, transform_eval])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train_epoch(X, y):\n",
    "    num_samples = X.shape[0]\n",
    "    num_batches = int(np.ceil(num_samples / float(BATCH_SIZE)))\n",
    "    costs = []\n",
    "    correct = 0\n",
    "    for i in range(num_batches):\n",
    "        idx = range(i*BATCH_SIZE, np.minimum((i+1)*BATCH_SIZE, num_samples))\n",
    "        X_batch = X[idx]\n",
    "        y_batch = y[idx]\n",
    "        cost_batch, output_train = train(X_batch, y_batch)\n",
    "        costs += [cost_batch]\n",
    "        preds = np.argmax(output_train, axis=-1)\n",
    "        correct += np.sum(y_batch == preds)\n",
    "\n",
    "    return np.mean(costs), correct / float(num_samples)\n",
    "\n",
    "\n",
    "def eval_epoch(X, y):\n",
    "    output_eval, transform_eval = eval(X)\n",
    "    preds = np.argmax(output_eval, axis=-1)\n",
    "    acc = np.mean(preds == y)\n",
    "    return acc, transform_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: Train cost 1.72300577164, Train acc 0.38824, val acc 0.6114, test acc 0.6087\n",
      "Epoch 1: Train cost 0.867130100727, Train acc 0.71758, val acc 0.7745, test acc 0.7759\n",
      "Epoch 2: Train cost 0.618825733662, Train acc 0.79848, val acc 0.8199, test acc 0.827\n",
      "Epoch 3: Train cost 0.475057393312, Train acc 0.8489, val acc 0.8602, test acc 0.8613\n",
      "Epoch 4: Train cost 0.369837403297, Train acc 0.88208, val acc 0.8697, test acc 0.8723\n",
      "Epoch 5: Train cost 0.336995840073, Train acc 0.89126, val acc 0.8957, test acc 0.8974\n",
      "Epoch 6: Train cost 0.288021206856, Train acc 0.90742, val acc 0.9005, test acc 0.8993\n",
      "Epoch 7: Train cost 0.260697960854, Train acc 0.915, val acc 0.9081, test acc 0.9091\n",
      "Epoch 8: Train cost 0.235620766878, Train acc 0.92484, val acc 0.917, test acc 0.9214\n",
      "Epoch 9: Train cost 0.232491567731, Train acc 0.9245, val acc 0.9205, test acc 0.921\n",
      "Epoch 10: Train cost 0.214803680778, Train acc 0.92916, val acc 0.9249, test acc 0.926\n",
      "Epoch 11: Train cost 0.191879570484, Train acc 0.93728, val acc 0.9306, test acc 0.9317\n",
      "Epoch 12: Train cost 0.187945634127, Train acc 0.93854, val acc 0.9365, test acc 0.937\n",
      "Epoch 13: Train cost 0.177504748106, Train acc 0.94238, val acc 0.9329, test acc 0.933\n",
      "Epoch 14: Train cost 0.161393344402, Train acc 0.9479, val acc 0.9246, test acc 0.9269\n",
      "Epoch 15: Train cost 0.158181488514, Train acc 0.9482, val acc 0.9353, test acc 0.9382\n",
      "Epoch 16: Train cost 0.162177875638, Train acc 0.94768, val acc 0.9399, test acc 0.9385\n",
      "Epoch 17: Train cost 0.150974154472, Train acc 0.95074, val acc 0.9417, test acc 0.944\n",
      "Epoch 18: Train cost 0.13878442347, Train acc 0.9546, val acc 0.9514, test acc 0.9481\n",
      "New LR: 0.000700000033248\n",
      "Epoch 19: Train cost 0.139381811023, Train acc 0.95302, val acc 0.9465, test acc 0.9477\n",
      "Epoch 20: Train cost 0.115818083286, Train acc 0.96186, val acc 0.9498, test acc 0.9515\n",
      "Epoch 21: Train cost 0.10844618082, Train acc 0.96364, val acc 0.9537, test acc 0.9544\n",
      "Epoch 22: Train cost 0.104168988764, Train acc 0.9651, val acc 0.95, test acc 0.9522\n",
      "Epoch 23: Train cost 0.100386917591, Train acc 0.96664, val acc 0.9523, test acc 0.9533\n",
      "Epoch 24: Train cost 0.101429723203, Train acc 0.9666, val acc 0.9516, test acc 0.9557\n",
      "Epoch 25: Train cost 0.0968987718225, Train acc 0.96804, val acc 0.9523, test acc 0.9556\n",
      "Epoch 26: Train cost 0.0905688554049, Train acc 0.97016, val acc 0.955, test acc 0.9533\n",
      "Epoch 27: Train cost 0.0892679914832, Train acc 0.97024, val acc 0.9574, test acc 0.9537\n",
      "Epoch 28: Train cost 0.0790596753359, Train acc 0.9733, val acc 0.956, test acc 0.9577\n",
      "Epoch 29: Train cost 0.0846520811319, Train acc 0.97228, val acc 0.9586, test acc 0.9575\n",
      "Epoch 30: Train cost 0.0861563980579, Train acc 0.9711, val acc 0.9553, test acc 0.9579\n",
      "Epoch 31: Train cost 0.084160938859, Train acc 0.9713, val acc 0.9574, test acc 0.9565\n",
      "Epoch 32: Train cost 0.0740946382284, Train acc 0.97538, val acc 0.9583, test acc 0.9568\n",
      "Epoch 33: Train cost 0.0750161111355, Train acc 0.97476, val acc 0.9522, test acc 0.9558\n",
      "Epoch 34: Train cost 0.0719307512045, Train acc 0.97592, val acc 0.9534, test acc 0.9601\n",
      "Epoch 35: Train cost 0.0688360854983, Train acc 0.97742, val acc 0.9568, test acc 0.9578\n",
      "Epoch 36: Train cost 0.0659850463271, Train acc 0.97732, val acc 0.9586, test acc 0.9602\n",
      "Epoch 37: Train cost 0.0669036284089, Train acc 0.97736, val acc 0.9606, test acc 0.9581\n",
      "Epoch 38: Train cost 0.0615548193455, Train acc 0.9792, val acc 0.9584, test acc 0.9538\n",
      "New LR: 0.000490000023274\n",
      "Epoch 39: Train cost 0.0617390647531, Train acc 0.9795, val acc 0.9585, test acc 0.9574\n",
      "Epoch 40: Train cost 0.0535897053778, Train acc 0.9818, val acc 0.9563, test acc 0.9582\n",
      "Epoch 41: Train cost 0.0471548065543, Train acc 0.98434, val acc 0.9622, test acc 0.9613\n",
      "Epoch 42: Train cost 0.0408403426409, Train acc 0.98648, val acc 0.9635, test acc 0.9624\n",
      "Epoch 43: Train cost 0.0405819378793, Train acc 0.98642, val acc 0.9636, test acc 0.9619\n",
      "Epoch 44: Train cost 0.0374028384686, Train acc 0.98754, val acc 0.9606, test acc 0.9614\n",
      "Epoch 45: Train cost 0.0365789830685, Train acc 0.98828, val acc 0.9591, test acc 0.9574\n",
      "Epoch 46: Train cost 0.0347327440977, Train acc 0.98848, val acc 0.962, test acc 0.9613\n"
     ]
    }
   ],
   "source": [
    "valid_accs, train_accs, test_accs = [], [], []\n",
    "try:\n",
    "    for n in range(NUM_EPOCHS):\n",
    "        train_cost, train_acc = train_epoch(data['X_train'], data['y_train'])\n",
    "        valid_acc, valid_trainsform = eval_epoch(data['X_valid'], data['y_valid'])\n",
    "        test_acc, test_transform = eval_epoch(data['X_test'], data['y_test'])\n",
    "        valid_accs += [valid_acc]\n",
    "        test_accs += [test_acc]\n",
    "        train_accs += [train_acc]\n",
    "\n",
    "        if (n+1) % 20 == 0:\n",
    "            new_lr = sh_lr.get_value() * 0.7\n",
    "            print \"New LR:\", new_lr\n",
    "            sh_lr.set_value(lasagne.utils.floatX(new_lr))\n",
    "\n",
    "        print \"Epoch {0}: Train cost {1}, Train acc {2}, val acc {3}, test acc {4}\".format(\n",
    "                n, train_cost, train_acc, valid_acc, test_acc)\n",
    "except KeyboardInterrupt:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAIvCAYAAABjmkFbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcZFV9///XZ2bYFRBZRBYHBURQERfQuA2IiCu4RMFf\noon7QtTEDRNjTSnfuCSuISoookYjRjEibqDg4IYggqJssioMCMgiMMPADPP5/XFuM9U11T1dPd19\nq+q+no/HfVTXvbdufbqb6XpzzrnnRGYiSZI0zObVXYAkSdL6MtBIkqShZ6CRJElDz0AjSZKGnoFG\nkiQNPQONJEkaerUHmog4OCIujohLI+KdPY6/LSLOq7bfRsSqiNiyjlolSdJgijrnoYmI+cAlwIHA\nUuCXwOGZedEE5z8HeEtmHjh3VUqSpEFXdwvNvsBlmXlVZq4ETgAOmeT8lwJfmZPKJEnS0Kg70OwA\nXN3x/Jpq31oiYlPgGcCJc1CXJEkaInUHmn76u54L/DQzb52tYiRJ0nBaUPP7LwV26ni+E6WVppfD\nmKS7KSJclEqSpBGSmTHVc+seFLyAMij4acC1wNn0GBQcEVsAVwA7ZuadE1wr+/nGNbsiYnFmLq67\nDq3h72Sw+PsYPP5OBku/n+u1ttBk5qqIOAI4BZgPHJeZF0XEa6vjx1SnHgqcMlGYkSRJzVZ3lxOZ\n+T3ge137jul6/gXgC3NZlyRJGh51DwrW6FpSdwFay5K6C9A4S+ouQGtZUncBmr5ax9DMJMfQSJI0\nOvr9XLeFRpIkDT0DjSRJGnoGGkmSNPQMNJIkaejVftu2JM0VZxSXBsNs3MRjoJHUKN4NKdVrtv7H\nwi4nSZI09Aw0kiRp6BloJEnS0DPQSJKkoWegkSRJQ89AI0mShp6BRpI0JyJidUT8aAausyQiVs9E\nTRodBhpJaogqUPSzvXwWypiJOUhyhq4zYyJi0VR+pnXXOcqcWE+SmqPN+CAQwFuALYCPAbd2nX/e\nDL//HsDyGbjOy4BNZuA6s+Eq4PM119BIkTlQIXfaIiKdAVTSZPw7sbaIuArYCdglM/9YczlDKyIW\nAacDSzLzgJrLGWhT/XfY779Xu5wkSWsZG6cSERtExHsi4pKIWBERx1fHN4+It0fE6RFxTUTcFRE3\nRMRJEfH4Ca651hiaiFhc7X9qRLwoIs6OiGURcVNEfCUiHjhRbV37xrp8WhHxqIj4TkTcWl1rSUQ8\nYYKato+I46val0fEeRHxss7rTf+nOLkp/IwnPV6d85iIOLGqf0VEXBUR/xURD+jxfp+vrrdLRPxD\nRJxffc/rPa5pENjlJEmazDeAxwLfrb6+odq/J3AUcAZwMnAL8CDgecAzI+K5mXlKj+tN1C3whuq1\nJwE/Ah4PvATYOyIelZl3T/E6jwXeAfwcOLaq6YXAadV1fj92YkRsC5wJ7Fx9Hz8Htgc+CfxgHe8z\nkyb6GU96PCKeA5xY1fh14A/Vea8HDomIJ2XmVT3e7+PAk4FvV9s9M/vt1MNAI0mazE7AXpl5c9f+\nC4Htu/dHxA7A2cBHKaFnqp4BPDYzL+i41peBw4FDgK9N8TrPBv4uM7/YcZ3XAJ8G3gy8sePc91PC\nzAcz810d53+s+h6mY5eIWDzBsYsy86s99k/0M57weETcB/gCpadlUWb+rOPYO4APAMdQfq7d9gEe\nlZl/WNc3M0zscpIkTeZfe33QZuZtE+xfSmk12CMiduzjfT7RGWYqn6keH9fHdX7aGWYqn6O0Qtx7\nnYjYkBKWbqW0NN0rM88Huq8xVQ8C3jPB9pIJXtPzZ7yO44cA9wO+2hlmKh+mtNY8PSJ26nG9D41a\nmAEDjSStUwQ511vd33MlmaSlIiKeGBH/GxFXV+M3xm5NPqI6ZYc+3uucHvuuqR7vtz7XycxVwPVd\n13kosDFwfmYu63Gd7pAwVUsyc94E2wt6nD/pz3iS44+uHk9f6wWZ9wA/rp7u0+O10219Gmh2OUnS\nOmTS2DujMvP6Xvsj4vmUcRvLKeNNLgeWAauB/YGnAhv18Vbdt4wDrKoe56/ndcau1XmdLarHnt/f\nJPtn3EQ/43UcH6v/ugledl3XeZ3+NMXShoqBRpI0He8DVlDGvVzSeaAaR/PUWqqautuqx+0mOD7R\n/kHxl+pxrbuZKtt3nddpUFoAZ5RdTpKk6dgVuLBHmJkHPKmekvpyESWQPbIaYNtt0L+Hc6vH/bsP\nRMQCyl1M2XHeyDPQSJKm40pg94gYawkgIgJYDDyMAW8FyMyVwAnAlsC7O49FxN6U2YgH2TeBm4HD\nI2K/rmNvARYCP8zMa7pfOKrscpIkTTRGaLKxQx+l3Ap9XkR8A1gJPJESZk4GnjujFfZX21QdCRwA\nvKMKBWdSumr+GvgOcChlTFA/JrttG+CjmdnZDbSu76Pn8cxcFhGvoNzOfkZEfA24GngM8HTKGJrX\nTrnqEWCgkaRmm2ihx0kXgMzMYyPiLkprwMsog4N/ArwceBHwnPV8/9l4zfgdmTdExF8B/wY8C9gP\nuJgyMd1ySqC5rft167j+zpRbtCc653OsGdeyru9jXb+Db0XEE4F/psw3swUlyHwKeF9mdg/+HbhF\nPWeSazlJagz/TmiqIuL/Ae8CnpGZP1jX+Zq62VrLyUAjqTH8O6FuEfHAzLy2a98jKMsgrAB26LHs\ngtbDbAUau5wkSU12TkRcClxAmUdnN8ryCQCvNswMD1toJDWGfyfULSLeQxkrsxC4L2WRzV8A/5GZ\nP57kpZomu5zWwT9UktbFvxNS/WYr0DgPjSRJGnoGGkmSNPQMNJIkaegZaCRJ0tAz0EiSpKFnoJEk\nSUPPQCNJkoaegUaSJA09A40kSRp6BhpJ0oyKiM9HxOqI2Llj38Jq3/F9XOfvqte8fHYqvfd91qpX\nw8dAI0kNERFfrj64Xz+Fc0+tzj1kmm830bo601lvZ73W6ImIxdX38pRJrj9Q6wB1hKzJtimHwyZw\ntW1Jao5jgcOBVwGfmuikiFgIHAhcC5w8zffqXoPnGmAP4C/TvN5sehfwfsr3O2i+Cfx6gmMT7W8k\nA40kNURmnhERvwf2iYh9MvO8CU59ZfV4fGaunqH3XgX8fiautR56LnSYmX8C/jTHtUzVNzPzi3UX\nMQzscpKkZvlM9fjqXgcjYj7w98Bq4LPVvkMj4ksR8fuIuKPazomIf4iIKa2GPNkYmojYNSK+FhG3\nVNf+WUQ8a5Jr7R8Rx0bEhRHxl4hYHhG/jYj3RMRGXedeBbynevqjzi6bjnMmHEMTES+OiB93vM/5\nEXFkRGzY49yrIuLKiNg0Iv49Iv4YESsi4tKIeMdUfk7T1fnzjYjdI+KrEXFDRNwTEU9Zx/GnVteY\nFxGvi4hfRsTt1e/i7GrfWr/n6no/iojtIuKzEbE0IlbN9pinidhCI0nN8gXg34DDIuKtmXln1/Fn\nAg8ETs3MP1T73g/cA5wJLAW2AJ4GfBx4HPCyPt5/3FiViNituu5WwHcp3Si7UbpavjfBNd4BPBT4\nOaVLbGPgScBiYFFEHNjRsvRR4FDgqcDngaumUldV278BRwI3Al8C7gCeRfn5PSMiDsrMlV3X2AA4\nFdge+A6wCng+8IGI2Dgz3zvB+8+UhwC/AC4B/hvYBLhtHcfHugH/m9Il+UdK8E3gBcAnKT/fv+nx\nfltV17sd+DolCNfT2pWZI7GVb6X+Otzc3AZ38+/EvT+HE6oPnpf3OHZSdewFHft26XFeUALCamDf\nrmNj+3fu2Lew2ve5rnNPrfb/Q9f+51X7VwMv6zq2Vj3V/vdW57+4a//iav9TJnhdr3qfUO27Cti2\nY/984FvVsXd1Xeeqav+3gY069m8D3FJtC6b4Oxqr6f+q+nttD+3x810NHNXjeus6fnh17Bxg0479\nmwK/rI4d3vWaset9HpjXx39/OZPnjW12OUlS8xxbPb6qc2dEbE9pgbieEmwAyMwruy+Q5RPnE9XT\ng6ZTRETsSBl8fAVwdNf1vwWc0et1veqpfGx96unyiurxqMy8oeO97wHeSvkgf1WP1yXwpsy8q+M1\nN1JC0BbA7n3WcQily6x7+1dKK1W3PwHtSa430fGx7/fIzFzeUfty4J3V017f713A23KGxlqtD7uc\nJGkdoh1zfktvtnJKY1Omde3M0yPicuCJEbFHZl5cHfp7SgvE56sPbgAi4v7A2ylh58GU/2vvtMM0\nS9mnevxpFZC6nUHpKhonIjYD3kzpytkduA/jB/xOt55Oj6aEk9O7D2TmpRGxFFgYEffNzNs7Dv8l\nM6/ocb2rq8f79VnH32V/g4J/k+O7waZ6/NGUbsUlPY79mBLgHtXj2FWZ+ec+6ps1BhpJWofZDBc1\n+ixlbMyrgLdVgz5fSfngGhs4TERsSelyWAicReleuJkyNuR+lGAxbiBuH7aoHq+f4PhaYzEiYgNK\nyHgc8FvgK5QxLispoaa1HvX0qu26CY5fB+wIbEkZPzLm1gnOX1U9zl//0ia1rvErEx3fArgpy91o\n42Tmqoj4M7D1NN5vzhhoJKmZjgfeB/xtRLwLeAqwC3BaVwvDqyhhZnF2DWiNiCdQAs10jQ1G3W6C\n4w/ose8QSpg5PjNf2Xmg6jJrrUc9vWrbntIl1m37rvMGxbpaEyc6/hdgq4iY39k6BxARCyhh5rYe\nrxuYCQkdQyNJDVSNCzmJMmD1UNaMjzi269Rdq8cTe1xmre6gPp1bPT4pInp9Hi3qsW+snm/0Uc/Y\nB3Q/rSPnUlp81qohInaltM5cmZm9PuSH0bmUn0+vn+FTKHnh3B7HBoaBRpKaa6xr6a2UUHMj5a6a\nTmMDcPfv3BkR+1Bm2J22zFwK/IAyLueIrusfQvkg7TZRPQ8GPjjBW91UPT6oj/I+Vz2+OyLu7Wqp\n5un5D0rYOa6P6w26se/3/RGxydjOiNgU+ED1dKC/X7ucJKmhMvPUauK5fatdX+gxhuKLlAHBH4uI\n/YHLKPPEPJvSanPYepbxRso8NB+LiIOA8ymtMIdS5ph5btf5J1c1/FNEPIIyb83OVT3fnqCe0ylj\ng95fveYWgMw8aqKiMvPMiPgQZc6b30XE14HllHl69gJ+Avz7dL7hPj2/Cmu9XJmZX5iJN8nMr1Qh\n8sXABRFxEqU76VBKl+MJmfmVmXiv2WKgkaRm+yxwFOXD6zPdBzPzuoh4MuX/0p8EPAO4CHg9cBq9\nA8SUF3vMzMsi4vHV9Q+kdPH8hjJWZlvgOV3nL4+IA6rzFwFPBi6nzEHz0V71ZObF1ey1b6vq3riq\nbyzQ9Kw3M4+MiPMorUcvo0yadxnwL8CHe4S/yb7nfhfAHDv3eZSfRS9LKBMlzpTDKXeWvQJ4TVXD\nRcC/Z+aEa38Niuh9p9zwiYjMHMk7ESTNEP9OSPWb6r/Dfv+9OoZGkiQNPQONJEkaegYaSZI09Aw0\nkiRp6NUeaCLi4Ii4OCIujYh3TnDOoog4LyJ+FxFL5rhESZI04Gq9y6maoOgSyq16SynrhRyemRd1\nnLMl8DPgGZl5TURs3WshLO9ekLQu/p2Q6jeqdzntC1yWmVdVq3+ewNr3278UODEzrwEYlFU9JUnS\n4Kg70OzAmiXVAa5h7WXfd6MsmPWjiDgnIv52zqqTJElDoe6ZgqfS37UB8GjgacCmwJkR8YvMvLT7\nxIhY3PF0SWYumYkiJUnS7IqIRfRekHRK6g40S4GdOp7vRGml6XQ18OfMvBO4MyJ+DOwNrBVoMnPx\nLNUpSZJmUdUIsWTseUS0+nl93V1O5wC7RcTCiNgQeAnwra5zTqIsLT+/WvVzP+DCOa5TkiQNsFpb\naDJzVUQcAZwCzAeOy8yLIuK11fFjqkXFvk9ZgXU18JnMNNBImpaIGI0F7CSNM1KLU0LOy+xrNVNJ\nkjSAhu227Zm2Ud0FSJKkuTdqgWbTuguQJElzb9QCzWZ1FyBJkuaegUaSJA09A40kSRp6BhpJkjT0\nDDSSJGnoGWgkSdLQM9BIkqShN2qBxnloJElqoFELNLbQSJLUQAYaSZI09Aw0kiRp6BloJEnS0DPQ\nSJKkoWegkSRJQ89AI0mSht6oBRrnoZEkqYFGLdDYQiNJUgMZaCRJ0tAz0EiSpKFnoJEkSUPPQCNJ\nkobeqAWajSKYX3cRkiRpbo1aoLkTb92WJKlxRi3QLMNuJ0mSGmcUA40tNJIkNcwoBhpbaCRJahgD\njSRJGnoGGkmSNPQMNJIkaeiNWqBZjoFGkqTGGbVAYwuNJEkNZKCRJElDbxQDjfPQSJLUMKMYaGyh\nkSSpYQw0kiRp6BloJEnS0DPQSJKkoWegkSRJQ2/UAo0T60mS1ECjFmhsoZEkqYEMNJIkaeiNYqBx\nYj1JkhpmFAONLTSSJDWMgUaSJA29kQw0EUTdhUiSpLkzUoEmk5XAamDDumuRJElzZ6QCTcVuJ0mS\nGmYUA42T60mS1DCjGGhsoZEkqWFGNdA4F40kSQ0yqoHGFhpJkhrEQCNJkoaegUaSJA09A40kSRp6\nBhpJkjT0RjHQOA+NJEkNM4qBxhYaSZIaZlQDjfPQSJLUIKMaaGyhkSSpQQw0kiRp6BloJEnS0Ks9\n0ETEwRFxcURcGhHv7HF8UUT8JSLOq7Z3r+OSBhpJkhpmQZ1vHhHzgaOBA4GlwC8j4luZeVHXqWdk\n5vOmeFkDjSRJDVN3C82+wGWZeVVmrgROAA7pcV70cU0DjSRJDVN3oNkBuLrj+TXVvk4J/FVE/CYi\nvhsRe67jmk6sJ0lSw9Ta5UQJK+tyLrBTZi6PiGcC3wR273ViRCyGnbaElz4w4oOLMnPJzJUqSZJm\nS0QsAhZN+/WZU8kUsyMiHg8szsyDq+fvAlZn5gcnec2VwGMy8+au/ZmZEcG2wAWZbDObtUuSpNkz\n9rk+1fPr7nI6B9gtIhZGxIbAS4BvdZ4QEdtFRFRf70sJYTevfal7OYZGkqSGqbXLKTNXRcQRwCnA\nfOC4zLwoIl5bHT8GeBHw+ohYRRkfc9g6LnsnsHEE8zJZPYvlS5KkAVFrl9NM6myaimAZsF0md9Rc\nliRJmoZh63KaLXY7SZLUIAYaSZI09Aw0kiRp6I1qoHFyPUmSGmRUA80yYNO6i5AkSXNjlAONLTSS\nJDWEgUaSJA09A40kSRp6BhpJkjT0DDSSJGnoGWgkSdLQG9VA4zw0kiQ1yKgGGltoJElqkFEONE6s\nJ0lSQ4xyoLGFRpKkhjDQSJKkoWegkSRJQ89AI0mShp6BRpIkDT0DjSRJGnqjGmicWE+SpAYZ1UDj\nPDSSJDXISAaaTO4GiGDDumuRJEmzbyQDTcVxNJIkNYSBRpIkDT0DjSRJGnoGGkmSNPQMNJIkaegZ\naCRJ0tAb5UCzHOeikSSpEUY50NhCI0lSQxhoJEnS0DPQSJKkoWegkSRJQ89AI0mShp6BRpIkDT0D\njSRJGnqjHGiWY6CRJKkRRjnQLMOJ9SRJaoRRDzS20EiS1AAGGkmSNPQMNJIkaegZaCRJ0tAz0EiS\npKFnoJEkSUNvlAPNncDGESP9PUqSJEY40GSyGlgBbFJ3LZIkaXaNbKCp2O0kSVIDGGgkSdLQM9BI\nkqShZ6CRJElDz0AjSZKGnoFGkiQNPQONJEkaegYaSZI09EY90CwHNq27CEmSNLtGPdDYQiNJUgMY\naCRJ0tAz0EiSpKFnoJEkSUPPQCNJkoaegUaSJA292gNNRBwcERdHxKUR8c5JzntcRKyKiBf0cXkD\njSRJDVBroImI+cDRwMHAnsDhEfGwCc77IPB9IPp4C+ehkSSpAepuodkXuCwzr8rMlcAJwCE9zvsH\n4OvAjX1e3xYaSZIaoO5AswNwdcfza6p994qIHSgh51PVruzj+gYaSZIaoO5AM5Vw8jHgyMxMSndT\nP11OBhpJkhpgQc3vvxTYqeP5TpRWmk6PAU6ICICtgWdGxMrM/Fb3xSJiccfTJZC/wUAjSdLAi4hF\nwKJpv740fNQjIhYAlwBPA64FzgYOz8yLJjj/eODkzPxGj2OZmTF+HxsCyzLZYMaLlyRJs6bX5/pk\nam2hycxVEXEEcAowHzguMy+KiNdWx49Zv+tzd0QJNpncPQMlS5KkAVRrC81MmijJRXArsEsmt9RQ\nliRJmoZ+W2jqHhQ8F5bhXDSSJI20JgSa5TgwWJKkkdaEQOOt25IkjTgDjSRJGnoGGkmSNPQMNJIk\naegZaCRJ0tAz0EiSpKFnoJEkSUOvCYFmOU6sJ0nSSJtyoImI1RHxP7NZzPqKdvRam8oWGkmSRlw/\nLTS3A3+YrUJmyNY99hloJEkacf0EmvOAPWerkBmybY99BhpJkkZcP4HmA8CzIuKg2SpmBhhoJElq\noF5jTiayHfB94LsRcRJwNvAnILtPzMwvzkx5fTPQSJLUQP0EmuM7vn5+tfWSgIFGkiTNmX4CzSum\neN5aLTZzyEAjSVIDTTnQZObnZ7GOmTJRoHEeGkmSRtioTazXK9AsxxYaSZJGWj9dTgBExGbAC4BH\nAVsCfwHOBf4vM5fNbHl9s8tJkqQG6ivQRMSzgS8AW/U4fHNE/H1mnjwjlU2PgUaSpAaacqCJiEcD\nJwLzgS8Bp1Nu294e2B94KfC1iHhiZv5qFmqdiom6nDaJYF4mq+e6IEmSNPsic2o3JUXEicCzgf0z\n88wex/cDzgC+m5kvmNEqp1ZfspgVwNbZGt/1FcFyYJtM6u4SkyRJUxARmZkx1fP7GRT8ZOBrvcIM\nQGaeBXwNeFIf15xpNwDb9Nhvt5MkSSOsn0CzBfDHdZxzdXVeXW7AcTSSJDVOP4HmOmDfdZzzmOq8\nutyIgUaSpMbpJ9B8B3haRLwrIuZ3HoiI+RHxVuDpwHdnssA+TdZC4+R6kiSNqH5u2z4KOBT4f8Br\nIuInlNaYB1DGzexCuevpqJkusg8TBRon15MkaYT1s/TBdRHxJODTlJaYB3Wd8gPgdZl57QzW168b\ngAf22G+XkyRJI6yvifUy80rgGRGxI7APZQDwX4BzM3PpLNTXrxsoMxh3M9BIkjTC+plY70rKHDNv\nzMxrgGtmr6xp8y4nSZIaqJ9BwdtQWmMGmYFGkqQG6ifQXAA8ZLYKmSEGGkmSGqifQPNx4HkRsfds\nFTMDbgS2iXZ0T5VsoJEkaYT1Myh4KeVOpp9GxLHA2ZTbtNdaDCozfzwz5fUnW3lXtGMZsCVwS8eh\nZfS++0mSJI2AfgLNjzq+/sdJzkvKitx1Get26gw0zkMjSdII6yfQvHeK501t+e7ZMxZoLunYZ5eT\nJEkjrJ+J9RbPYh0zqdfAYAONJEkjbMqDgiPiyoj4r9ksZoYYaCRJaphRm4cGDDSSJDXOqM1DAwYa\nSZIaZ9TmoQEDjSRJjTNS89BUJgo0m9ZQiyRJmgOjPA9NJ1toJEkaYaM8D00nJ9aTJGmERWbd+WNm\nRERmZkQ75gF3AZtmK1eWYwSwEtgkk5V11ilJktZt7HN9quf3Myh4KGQrVwM3AVvfuy9J7HaSJGlk\nTRpoIuIpEfGgqV4sIvaOiJetf1nrzXE0kiQ1yLpaaJYAL+/cERHvjIibJzj/+cDxM1DX+jLQSJLU\nINPpctoE2HKS41Pu75pFBhpJkhpk5MbQVAw0kiQ1SNMCjZPrSZI0gpoWaGyhkSRpBDUp0Di5niRJ\nI2o6gWaymfgGZZY+W2gkSWqQqSx90IqIVsfzAIiIe3qcGwxGqDHQSJLUIFMJNBPdht3v/rlkoJEk\nqUEm7XLKzHnT2eaq+EksAyLasVnXPgONJEkjaBDCx4zLViZrt9IYaCRJGlEjGWgqvQKN89BIkjSC\nmhZobKGRJGkENSnQOA+NJEkjqkmBxhYaSZJGVO2BJiIOjoiLI+LSiHhnj+OHRMRvIuK8iPhVRBww\nxUsbaCRJaohaA01EzAeOBg4G9gQOj4iHdZ32w8zcOzP3Af4OOHaKlzfQSJLUEHW30OwLXJaZV2Xm\nSuAE4JDOEzJzWcfT+wB/nuK1b8RAI0lSI9QdaHYAru54fk21b5yIODQiLgK+B7xpite2hUaSpIao\nO9BMad2nzPxmZj4MeC7w31O8toFGkqSGmMpaTrNpKbBTx/OdKK00PWXmTyJiQUTcPzNv6j4eEYvv\nfbIxP+VIto52zMtWrqbctr1JBJE5EAtoSpKkSkQsAhZN+/WZ9X22R8QC4BLgacC1wNnA4Zl5Ucc5\nDwGuyMyMiEcDX8vMh/S4VmbmuIUxox23ALtmq4SfCJYDW2eyfNa+KUmStN56fa5PptYWmsxcFRFH\nAKcA84HjMvOiiHhtdfwY4IXAyyJiJXAHcFgfbzHW7TTWmjM2uZ6BRpKkEVJ3lxOZ+T3KYN/Ofcd0\nfP0h4EPTvPxYoBlr8RkbR3PjNK8nSZIGUN2DgmfbDcA2Hc8dGCxJ0ghqQqDxTidJkkacgUaSJA09\nA40kSRp6TQw0m9ZUiyRJmiVNDDS20EiSNGIMNJIkaeg1LdCMTawnSZJGyKgHmluA+0Y7Nqye3wg8\noMZ6JEnSLBjpQFMtSvlnYOtq1wXAXvVVJEmSZsNIB5pKZ7fT74BH1FiLJEmaBU0LNFcDm0bc22Ij\nSZJGQKMCTSZJaaWx20mSpBHSqEBTsdtJkqQR09RA8/CaapEkSbOgiYHmtxhoJEkaKU0MNBcAD48g\naqpHkiTNsMYFmkxuBFYAO9ZWkSRJmlGNCzQVu50kSRohjQk00Y7OLiYHBkuSNEJGPtBkK5cByfhF\nKb11W5KkETLygabinU6SJI2wpgaaC4E9IphfUz2SJGkGNTLQZHIH8Cdg19oqkiRJM6aRgaZit5Mk\nSSOiyYHGO50kSRoRTQ803ukkSdIIaHKgsctJkqQR0eRA83vgQRFsXEM9kiRpBjU20GRyN3A5sEct\nFUmSpBnT2EBTcRyNJEkjoCmB5s/A/aMd3d+v42gkSRoBjQg02cqVwG3AVl2HvHVbkqQR0IhAU7kR\nb92WJGkkNSnQ9BpHcyWwVQRb1FCPJEmaIY0ONJmspixUuVctFUmSpBnR6EBTsdtJkqQhZ6DxTidJ\nkoaegcY7nSRJGnpNCzTb9Nj/O+AREcQc1yNJkmZI0wJNrxaaP1WP281hLZIkaQY1PtBkktjtJEnS\nUGt8oKnb/kEoAAAgAElEQVQYaCRJGmJNCjS3AveJdmzY49hv8dZtSZKGVmMCTbZyNWX5g4kGBttC\nI0nSkGpMoKlM1O10AbBnRON+HpIkjYSmfYBPNDD4VuAWYOFcFyRJktafgWYNu50kSRpSTQw0E803\nY6CRJGlINS3Q/A7Yb5Jj3ukkSdIQalqg+Q5wULRjox7HXKRSkqQh1ahAk628ntISs3+PwxcDu0bQ\na54aSZI0wBoVaConAYd078zkTuCPwO5zXpEkSVovTQ00z4t29Pre7XaSJGkINS7QZCsvAW4HHtPj\nsHc6SZI0hBoXaCo9u50w0EiSNJQMNOO5SKUkSUOoqYHmLGDbaMeDu/ZfDmwfwWY11CRJkqapkYEm\nW3kPcDJdrTSZrAIuAfasoy5JkjQ9jQw0FbudJEkaEU0OND8E9ol23L9rvwODJUkaMo0NNNnKO4HT\ngWd3HTLQSJI0ZBobaCq9up3scpIkacgMRKCJiIMj4uKIuDQi3tnj+P8XEb+JiPMj4mcR8cgZeuvv\nAAdGOzbu2HcNsGkEW8/Qe0iSpFlWe6CJiPnA0cDBlLuLDo+Ih3WddgXwlMx8JPA+4NiZeO9s5Y3A\n+cAB9+5LktLttNdMvIckSZp9tQcaYF/gssy8KjNXAiew1u3UeWZm/qV6ehaw4wy+/0TdTo6jkSRp\nSAxCoNkBuLrj+TXVvom8EvjuDL5/r8UqzwMeP4PvIUmSZtGCugsAcqonRsT+wCuAJ05wfHHH0yWZ\nuWSdb97KS6MdtwCPo7T+APwAeG8E8zJZPdX6JEnS9ETEImDRdF8/CIFmKbBTx/OdKK0041QDgT8D\nHJyZt/S6UGYunmYNY91OZ5XrcEUEtwF7U1prJEnSLKoaIZaMPY+IVj+vH4Qup3OA3SJiYURsCLwE\n+FbnCRGxM/AN4G8y87JZqKHXOJpTgGfMwntJkqQZVnugycxVwBGUAHEh8NXMvCgiXhsRr61Oew9w\nP+BTEXFeRJw9w2WcDWwV7di1Y5+BRpKkIRGZUx7CMtAiIjMzpv36dhwLXJKt/HC5HvcBrgO2z+SO\nGSpTkiRNQb+f67W30AyQcd1OVYj5JbB/bRVJkqQpMdCscRqwd7Sjc4Zgu50kSRoCBppKtnIFZQXu\n53TsNtBIkjQEDDTjdd/tdD5w3wgeXFM9kiRpCgw0430HOCDasQlANaneqdhKI0nSQDPQdMhW3kSZ\nSO/Ajt12O0mSNOAMNGs7CXhex/MfAIsi2KCmeiRJ0joYaNZ2EvDcscUqM7kBuBx4Qq1VSZKkCRlo\numQrrwBuBPbr2P197HaSJGlgGWh6Owl4fsfzU4CDa6pFkiStg4Gmt/8FXjLW7QScCTwkgm1rrEmS\nJE3AQNPbb4E7qMbNZLKSsqT502usSZIkTcBA00O2MoGvAId37Pb2bUmSBpSBZmInAH8d7VhQPT8F\nOCjCn5kkSYPGD+cJZCsvA/4AHACQyRXA7cAj66xLkiStzUAzObudJEkaAgaayX0VOCTasVH13EAj\nSdIAMtBMIlt5LfAb4JnVrh8Bj4vgPvVVJUmSuhlo1u3ebqdM7gB+CSyqsyBJkjSegWbdTgQOjnaM\ntcrY7SRJ0oAx0KxDtvIm4KfAIdUuA40kSQPGQDM1nXc7nQ9sHsGDa6xHkiR1MNBMzUnAk6Md989k\nNXAqttJIkjQwDDRTkK28ndLV9MJql91OkiQNEAPN1HV2O/0AWBTBBjXWI0mSKgaaqfsesHe0Y4dM\nbgAup1qNW5Ik1ctAM0XZyhWUsTQvrnbZ7SRJ0oAw0PSns9vp+xhoJEkaCAaa/pwOPCjasStwJrBr\nBNvUXJMkSY1noOlDtnIV8DXgsExWAkuAp9dalCRJMtBMw1eAw6MdAXwHeEHN9UiS1HgGmv6dCdwH\neARwAnBABDvWW5IkSc1moOlTtnI1JcgclsntwJeB19ZblSRJzWagmZ4TgMOqbqf/Al4dwUY11yRJ\nUmMZaKbn18DdwH6ZXAz8FnhRvSVJktRcBpppyFYm4+ekORo4or6KJElqNgPN9H0FeHG0Yz7wbeCB\nETy25pokSWokA800ZSt/D1wLLMrkHuCTwBvrrUqSpGYy0KyfzwMfjXbsBRwHHBrB1vWWJElS8xho\n1s/RwH8CS1gcr2DeypOAV9ZckyRJjROZWXcNMyIiMjOjlvdux0LgeFZsvhWfPXNr/rznzlU3lCRJ\nmoZ+P9cNNDP1/u2YBxzBii0+zB+feDy7f/d11SR8kiSpTwaauuvY6+tv5antf2W7350L/H228g91\n1yRJ0rAx0NReBxsRq/7AGx/+Jba+5OXAu4DjqrlrJEnSFPT7ue6g4BmWyV3kgs9w9MUbAfsDbwC+\nE+14YM2lSZI0smyhmQXV6tvnAwtZHHcCR1HCzX621EiStG620AyATK4BTgP+Nlu5ktLttAB4Ya2F\nSZI0omyhmSURPBX4NLBnJhnteDplZe69qpAjSZImYAvN4PgxsAo4ACBb+QPgjzjxniRJM85AM0sy\nSdZehftI4D3Rjs3qqUqSpNFkoJldXwaeEsGDALKV51Babt5Sa1WSJI0YA80syuQO4IvA6zp2vxv4\nx2iHi1hKkjRDDDSz75PAKyPYGCBbeRlwAvAvtVYlSdIIMdDMskwuBX4FvLRj9/uAl0U7dqmnKkmS\nRouBZm68D/hABK0INsxWXg/8J/DemuuSJGkkGGjmQCY/B/YBHgucE8FjgQ8DT4927F1rcZIkjQAn\n1ptDEQSl6+kjwPH882bXs+Hyp2crn1VzaZIkDRRX2x4CEWwHHM38FY/gyPvdlw1W/E228kd11yVJ\n0qBwpuAhkMn1mfw192z8L3z705tyyy5fic1ucLI9SZKmyUBTo0xO5PKDdiPZgF1PuTyiLJMgSZL6\nY6CpWd6+/Z/Z6srDePYb7mHe3V+I4IPVWBtJkjRFBpoBkK38ARvdcSH/tNNHgGcB/1h3TZIkDRMD\nzeA4kvvc8HYecsqLgH+K4IV1FyRJ0rAYiEATEQdHxMURcWlEvLPH8T0i4syIWBERb62jxtmWrfwV\n8GP+9uC3sc9xrwI+HcHj665LkqRhUPtt2xExH7gEOBBYCvwSODwzL+o4ZxvgQcChwC2Z+eEe1xma\n27YnEu14IHAUcAjLtrmOnxy5I6s3eHqe9Q+/rLs2SZLm0tDNQxMRTwBamXlw9fxIgMz8QI9zW8Ad\noxpoxkQ7NgQO4rpHvYetLn8M8+/6MQvu/hLwf9nKm+uuT5Kk2dbv5/qC2SxminYAru54fg2wX021\nDIRs5d3At4FvxyY3f4w9T3w6z3n9s5l3z0eiHT+hrNZ9Urby9norlSRpMAxCoJmxJqKIWNzxdElm\nLpmpa9dmxVb/xLmv/l/OffVdvHOrHdnklucBhwEfjXZ8CPjPbOWKmquUJGm9RMQiYNG0Xz8AXU6P\nBxZ3dDm9C1idmR/scW4jupy6RbAJcDrww0z+FSDasQfwfuDRwHuAL2Ur76mvSkmSZs4wLn1wDrBb\nRCyMiA2BlwDfmuDckQws65LJncAhwEsjeAVAtvLibOXzKYtdvgY4L9rxzGhHI39GkqRmq72FBiAi\nngl8DJgPHJeZ74+I1wJk5jER8QDK3U+bA6uB24E9M/OOjmuMbAvNmAgeCpwB/E0mP7x3fwkxhwAf\nAK4F3pGtPKeeKiVJWn9Dd5fTTGlCoAGI4CnA14EDMvnduGPtWAC8AmgBPwH+JVt5+dxXKUnS+jHQ\nNEAEhwFHA58APpLJHeOOt2MzyvIJbwH+G3iPd0RJkobJMI6hUZ8yOQHYF3go8PsIXh/BBvceb+Wy\nbOVRwMOALYALoh3PradaSZJmny00Qy6CRwMfpMyk/M/AiZnjb4WPdhwAHAOcB7wpW/mnOS9UkqQ+\n2OXUUBE8nRJsVgLvyOSMccfbsQnwr8CrgH8BjstWrp7zQiVJmgIDTYNFMA84nLIe1AXAkT0GDj8S\n+AywAnhNtvKSOS9UkqR1cAxNg2WyOpMvA3sAPwROi+DYCDa/95xWng/8FXAi8LNox7urtaMkSRpa\nttCMsAi2AP6dspL532bys3HH27Ez8ElgIfBm4Ixs5aq5rlOSpG52OWktERxCGRT8GeC9may891iZ\nlO+vKeNqFgI/B5ZU268MOJKkOhho1FMEDwA+B2xDmWl4rbEz0Y77A0+hLA62iLUDzrnZypXdr5Mk\naaYZaDShCAJ4PdCm3PF0TPct3uPOXzvg7EKZgfhbwEne/i1Jmi0GGq1TBHsAXwauA16ZyfVTel0J\nOAdS1o16JnAR8H/AN7OVl85SuZKkBjLQaEoi2JCy5tMrgNdkcnJfr2/HRpRWm+dTAs5NwDcpAefc\nbI3If1iSpFoYaNSXCJ4MfJEyb83twIbARtVj99cbAncCJwP/C/wuk4x2zAP2Aw6lBJyNgW8An81W\njpsHZxBEO54KvAn4SLbyZ+s6X5I09ww06lt1e/dzgXuAuzu2u3o8vz/wAsqdUXcCX6u231bhJihr\nSB0GvBK4Avg0cGK2csUcfltriXbsSLmN/a8od329GfgQJdiMxj8ESRoRBhrNiWqA8eOAF1PCzQpK\nq01nuNmAEpReBzwK+AJwTLbysjmttXSPvQV4O/Ap4P3ZyuXRjoVVzdcAr8hW3jqXdUmSJmag0Zzr\nCDd/XW13Af8DfDST2wCiHbsCrwH+Dvg1pdXm5Nm+DTzacTDwceAS4B+zlZd3Hd8I+DBwMPCibOWv\nZ7MeSdLUGGhUq45w8wbgIMqEfV/IZDVAtGNj4IWUVpuHAJ+ltNosndE62rEL8FFgL+DN2crvruP8\nw4FPAO+iLNw5Gv8wJGlIGWg0MCJ4HCUkLADelMmZ44634+GUeXEOp6w9dTTwk/UJE9GOTYF3Am+k\ntLx8JFt51xRf+zDg68AvgTdkK5dPtw5J0vox0GigVCuAvxT4APAjygrg41pjoh2bAy8DjqAMPj4a\n+HK2ctmU3qMd21JuIT8AeA7wM+Bt2cqr+663HZtRBgzvTemCcjVySaqBgUYDKYL7ULpzXgd8BPhw\nJuPueqrukHoaJdg8mTKI+JPdg4ijHVsCT6UEmAOAnYAfA6cDP1zfW8WrOl4DHFXV8r92QUnS3DLQ\naKBF8GDgPyh3Pb0V+Gav5ReqO5BeR5n47xzgq8CelACzB2WNqdOr7bzZWEQz2vEY4EvAfEpX1Ner\n9xqNfzSSNMAMNBoKETwN+BhwB/A74Pqe2z/teCebL30JZTbiX1O6rc7KVt49J3WW1ppHAy+i3ME1\njzXh5peGG0maHQYaDY0IFlDuhNoJ2G6CbWPgBspcMWcApwI/y2RKA31ntN4Sbh7JmnCzCXAiJdz8\nIlu5eq5rkqRRZaDRSIlgY0qwWUjpbjqIciv2Tyjh5lTg4slWDZ+Vukq42YsSbl4EbAWcD1zZY7vZ\nlhxJ6o+BRiMvgvtRBg8fBDyD0g10KnAKcFomN815TWXiwD2AXaptYcfX81gTbq4Ajs5WXjHXNUrS\nMDHQqFGqifx2p4Sbg4CnULqmPgN8L5MZHyzcr2jH/VgTbh5NuYPKCfwkaRIGGjVadXv4S4BXAzsC\nxwPHZXJVnXV1qiYU/BLwR+DV2crray5JkgZOv5/r82azGGmuZXJHJsdl8njgmcAWwDkRnBLBiyLY\nsOYSqebJ2Zdyd9evox2HTvda0Y6Nox3+O5bUeLbQaORFsAnwAkqrzR7AF4HPA0uBlZTZie+ZysDi\nCOZT7m7atOtxQ+D3mdzcV23teGJVzxnAW7KVt03hNRtTwtpLq8e7KLMj/5QyWPpXc3VbuyTNFruc\npElEsDvwKkq31JbABpQwMp814abzcSWwEWuCywbAcuDOrsdVlLB0LfAL4Kzq8bfrGscT7bgvZd2p\npwMvz1b+uMc584H9KSHmUOA3lBXNT6Tc2v6kjm134FeUcPNT4OdTCUqSNEgMNNI0VGtObcCagDP2\nuIDSAjIWXO6eqCWnar3ZE3h8x7YzcC4l3PwC+EUm1/V8fTueAxxLGV/zr5RQtR9l8c6XUObi+R/g\nq5OtTl6tjfUESrh5MvBY4FLKrMrfpywAumKi108m2rEAeAzlLrN5wIXVdnm2cuV0rilJvRhopAES\nwZbA4xgfcm5gzW3mZ2Ry7yKc0Y5tKKHmoZSWoVWUEPOVbOXvp1VDOzakhJADgYOBR1C6qL5f1XDJ\nRHdbVfPt7EEJMAdS1tD6I3AaJejtWW07ApezJuBcCFwAXJqtvLsa57MZcN+O7T5dzwFur7Y7Or4e\ne36HkxdKzWGgkQZY1RK0D2X+nIMoQeNsSrA4FfgNiwPgWcB19Fg7KoJNKbMr7wQ8kDL3zoQtNmvV\nUG4jf1pVw8HAataEm9MowWMswBxICVU/qI6d3uuurGjHJpSurj27toXV6zeltHJ1h5TO5zA+4HSH\nnk0prWRLgfMoLV/nVT+jP0/1+2+qaMd+wFsoy4j8hRKsJ9uuBa5xagHVxUAjDZEI7gssooSLZ1A+\nuE8Ffggka4JL57YppfvpauBmytw7i4FPZ3JPX+9fWmAexppw80RKy8vpVQ2nUbqTpvWHItqxEaXr\nbtn6tq5U44g2Ax5ECYWPrh73AW5lfMg5F1ja9A/jaMcGwAspQWY74BPAf1N+J9uuY9uZEkZ/3LFd\n1PSfqeaOgUYaYtVq5AdRlnlYRQkt3dufO8fxRLAnpZtqPvCaTH477fcvAWRVtrKvYFSnqjvrwawJ\nN2NBZwHl1vjO7YJs5ZTvRKvGDG0HbE8ZM3RBtnLZ5K+a8FpBmVxxbPD2Qko33e+r7RLgqplYOT7a\ncX/KBI5vqN7jY8DJ/fxeq3ofTAnMT60e78v4gHP+MP23Mplox4KZ+Nlr5hhopAaqurJeDRxFCTdH\nZXJnvVXVp/ow3o6y3tbDO7a9KN1bnSHnJkpgeWDH49jXWwN/pnT/BWVs09WUld9/U22/Bq5dq2uw\ntCg9kvF3oAVr7j67nBJwHkrprtu9et+rKOFmLOhcQWmJuxW4BbhtotauaMdewJspi6d+E/h4tvLX\nff74JhTt2Iky0Pwp1fZAymD339O1htkg3llX/XexDWVc2B6U1smxr3cGfg4cDXzDQe71M9BIDRbB\n9sDHKS0Ur8vktJpLGijVB9rOrAk3D6csLHotJbRc2/X1DZ3/11514TwUeBSwd/X4KEpQGQs5t1Hu\nMntCdY2xAPNTygf9hH90qzmGHsL4kLMLcL9q25Iyrug21gScscetKB/MnwKOmYsZqKMd21IGuj+E\nNct7jG13sfZCrZdR7rj743RadqrxXw+lBJHdKHcjrp5gy+oxKK1hY8ElgIuAi7u2q4FnA0dU73EM\ncGy2suddiYOqalW8ZxS6Bg00kojgOcB/AT8C3paJg2ZnSRWStmdNwNkSOBP4Wbbyxll4vwXA5qwJ\nOGOPq4DvZSvvmun37FdHS8hC1gScBwO7Vtu2lJaoS1kTcsYer6G0/PRqRdmMNQHkUmAFpStwbIuu\n52OzaP+h43U3ruvDvlqe5A2UKRNOobTa/KzukBDt2AzYoWPbscfX21IG3J8D/LJjG7oxZQYaScC9\n61q9j/JH+e3AlzPxtmfVrror7sGUVpZdq8exr3cA/kTvVpQ5/VCOdmwBvBx4I+UuvaOB/8lWLp+D\n996cMofUftW2L6UVbikl9E30+KfqvMd1bfcwPuCcM+h3BxpoJI0TwWOBT1LGc1xNafq/qsfj9VNZ\n/kGaTdGO+YM20LgaeH4gJdg8kTLH0p+q7foeX9/QzxicqtXt4YwPL7tQujHPqrazKYPG+/432tHV\n2hlwHgMszlZ+tN/rzRUDjaSeqjWtdqb8oVzY4/E+lKb5qyn/t7eUMgZkacd2Q7+3hkujJNqxA6U1\n6QEd23Zdz7emzPWznDVjeXqN7xlrMV1I+Xd3NmsCzPmzOTC5CmkbZSsH9uYBA42kaam6qBaypi/+\ngazpnx/7eivKpGtjAWdsPpxrOrZrM6l9HIdUl+oOt/tT1llb1xifecAfspW31lPt4DLQSJo1EWxA\n+T/QsUGIvbbtKXfejAWd2yl/yDs3ejy/gzUtQtd2fH3Duhb4lDR6DDSSalUt0rktJdzsRLkzBUoz\ne07y9X1Z0xo0tu1A+T/dG1kTdP5AmadlbK6WaxzsLI0eA42kkVK1Cm3Hmu6vhayZp+WhwBaUW3jH\nAs5Y2LmONWMVJntcZguQNHgMNJIaJYLNWTMJ3UM7tm1Z06U1b5LH+ZT1n34xtmVyzQzXuCllXpZt\nOx43p9xKu6rjsdfXtwBnZXL3TNYkDToDjST1oQpEj6XMeDu2raQj4AC/ymR5df5GlG6w+1MGSXd/\nvTVrh5cFlMHUN3Y8/oUSqBZQQtWCrq/HHren3FVzGvAd4HuZDNXstdJ0GGgkaT1EMLaIZGfAeThl\nPaX7Uabbv6l6flOPr29ifHi5Abhjfeb4iWBbymroz6IsXnol8F1KwPmlt9JrFBloJGmGRbAxZRzP\nzaxnOJmBWhYAf0UJN8+m3HX2fcpaUQsog7DvUz12fj32OBbIbuzabuh6fpsTLapOBhpJapAIHgQ8\nkzK77ApgGeUW+M7Hzq9XUrrHxrrEtpng6w1Yc+v8RFPtX5eJq1JrVhhoJEnrrRrIPHbr/I70Xgxx\nW8pYoHvoPWFc96RysPbt+t238K+ihKUrKF1rV3R8fWUmy2bj+9XgMdBIkuZE1f11f0pY6Z7ev3uq\n/84Pm8kmWtyAEpYezJpVuse+XkgJUGPrj91BaXG6e5LHu6pzLwSuthtteBhoJEkjKYJ5lDFDD6aE\nm00pAWjDSR43rs7fkzJ544U9tj84OePgMdBIktRDBPcDHkYJN53b/YGLgd8C5wO/Ac7P5MaaShUG\nGgONJKkv1VxEDwMeAewNPLJ6XE5HwKm2ix0IPTcMNJIkradqPqKdGB9wHgk8iDJI+RJKq87YUhuX\nZHJLPdWOJgONJEmzpLr7azfK8hp7MH65jTsZH3LuYO0B0L22Oyhz//y54/GWpo/rMdBIkjTHqhad\n7VkTbnanTGTYfXt65zYWWO5LWTJjbNmMrat9tzA+6NwK3AbcXm23TfC4gjXrga21Ptiw3OlloJEk\nachVq8xvxZqAsw2wJSXobD7J4+bARky+RthqSrh5dyb/PmffVJ8MNJIkqaeqJWlsUdTVgzzAud/P\n9QWzWYwkSRocVXfTPdU2UubVXYAkSdL6qj3QRMTBEXFxRFwaEe+c4JxPVMd/ExH7zHWNkiRpsNUa\naCJiPnA0cDBltsbDI+JhXec8C9g1M3cDXgN8as4LVd8iYlHdNWg8fyeDxd/H4PF3MtzqbqHZF7gs\nM6/KzJXACcAhXec8D/gCQGaeBWwZEdvNbZmahkV1F6C1LKq7AI2zqO4CtJZFdReg6as70OwAXN3x\n/Jpq37rO2XGW65IkSUOk7kAz1XvGu2/bGo17zSVJ0oyo+7btpZS1MsbsRGmBmeycHat9a4kIg84A\niYhW3TVoPH8ng8Xfx+DxdzK86g405wC7RcRC4FrgJcDhXed8CzgCOCEiHg/cmpnXd1/ISfUkSWqu\nWgNNZq6KiCOAUyjTMR+XmRdFxGur48dk5ncj4lkRcRmwDPj7GkuWJEkDaGSWPpAkSc1V96Dg9TaV\nifk0uyLicxFxfUT8tmPfVhHxg4j4fUScGhFb1lljk0TEThHxo4i4ICJ+FxFvqvb7O6lJRGwcEWdF\nxK8j4sKIeH+1399JjSJifkScFxEnV8/9fdQoIq6KiPOr38nZ1b4p/06GOtBMZWI+zYnjKb+DTkcC\nP8jM3YHTqueaGyuBf8zMvYDHA2+s/l34O6lJZq4A9s/MRwGPBPaPiCfh76RubwYuZM2ds/4+6pXA\noszcJzP3rfZN+Xcy1IGGqU3Mp1mWmT8Bbunafe+EiNXjoXNaVINl5p8y89fV13cAF1Hmc/J3UqPM\nXF59uSFlzOAt+DupTUTsCDwL+Cxrpgbx91G/7ht8pvw7GfZAM5WJ+VSP7TruRrsecHbnGlR3EO4D\nnIW/k1pFxLyI+DXlZ/+jzLwAfyd1+ijwdmD1/9/e/Yf6VddxHH++HBtY5mws2gqDgWEF4rBi4Vbb\nQssgZ5BagTqEBOmfiiRKYs4KLIrynyCIEbnGLaOcI4VQJk3nH2O0MvyVkRWNNX/ANlGx6d79cc5t\nX758v/fu7m73dHafD/jyuXzO55z7+d7PP697zufzOQN1jke3Cngwyd4kN7V1JzwmXS/bni1nNPdA\nVZV7BM29JOcAvwa+WFUvJcf/8XFM5l5VHQNWJlkM/C7J+qHjjskcSfJJ4Lmq2jfu/U2ORydWV9WB\nJG8DHkjy1ODB6cak73doTmRjPnXjYJJlAEmWA8913J95JclCmjCztaq2t9WOyf+BqjoM3Ae8H8ek\nK5cCG5I8C0wAH02yFcejU1V1oC2fB+6hmVZywmPS90Dzv435kiyi2ZhvR8d9UmMHsLH9eSOwfYq2\nOoXS3IrZAjxRVXcOHHJMOpJk6eTqjCRnA5cD+3BMOlFVt1bV+VW1AvgssLOqrsfx6EySNyV5S/vz\nm4GPAX9mBmPS+31oknwCuJPjG/Pd0XGX5p0kE8BaYCnNM85NwL3A3cC7gL8D11bVoa76OJ+0q2d2\nAY9x/LHs14E9OCadSHIRzYTGs9rP1qr6XpIlOCadSrIW+EpVbXA8upNkBc1dGWimw2yrqjtmMia9\nDzSSJEl9f+QkSZJkoJEkSf1noJEkSb1noJEkSb1noJEkSb1noJEkSb1noJE0byXZnORYko903RdJ\ns2OgkXTS2jAw3cewIOm06/vLKSV1r4Dbpzj+j7nqiKT5y0Ajadaq6ptd90HS/OYjJ0lzZmDOytok\nG5PsS/JKkoNJtiR5+5jz3p3kriT7k7zWlj9LcsGY9guS3Jxkd5LD7e94JslPxpyTJFcn2ZPk5SQv\nJplI8o5T+geQdNp4h0ZSF75M8zbdXwD3Ax8GbgTWJVlVVS9MNkzyQeBB4Byal54+AbwXuA64Ksll\nVZM8X/UAAALLSURBVLV3oP0i4LfAZcA/gZ8DR4AVwKeAh4G/DvXnC8CG9voPAR8CPgNcnGRlVf3n\nlH57SaecgUbSbCXJbUBGHHu1qr47ov4KYFVV/WngIj8AvgR8B/j85IWBu2jCzHVVNTHQ/lqaQLQ1\nyfvq+Jt2N9OEmR3ANVV1dOCchcDiEf35OPCBqnp8oO024HPAVcCvpvwLSOqcb9uWdNKSHJumyaGq\nWjLQfjOwCdhSVTcNXetcmjsqC4HzqupoktU0d1Qerao1I37/LmANsLaqHk6yAHiR5p+1C6rq39P0\nf7I/366qTUPH1gE7ge9X1Ven+Z6SOuYcGkmzVVV11pjPkjHn/H7ERY4AfwTOpnmkBHBJW+4cc52H\n2nJlW74HOBd4bLowM2TviLp/teVbZ3AdSR0x0EjqwsEx9ZMhZPFQeWBM+8n684bK/TPsz6ERda+3\n5YIZXktSBww0krowcjUTsKwtDw+Vy0a0BVg+1G4ymLzz5LsmqY8MNJK6sG64IslimkdHrwJPttV/\naMv1Y66zfqjdUzTh5uIky0efIulMZKCR1IXrk6wcqttMM/9lYnJlUlXtBp4G1iT59GDjJFfTTAh+\nuqoeadu/AfyIZh7Oj9sl3IPnLEqy9DR8H0kdc9m2pNmaatk2wD2Dy7Nb9wO7k9xNM29mDbAaeBb4\n2lDbjcADwC+T3EsTcC6k2VPmCHDDUPvbgVXAlcBfktwHvAScD1wO3EKzFFzSGcRAI2m2CrhtimN/\nA4YDzQ+B7TT7zlxIEzh+Ctw6uKkeQFXtaTfX+wbN/jJXAs8D24BvVdUzQ+2PJrkCuJkm7NxAE7b2\nA78BHhnqn3tXSGcA96GRNGcG9n1ZV1W7Ou6OpDOIc2gkSVLvGWgkSVLvGWgkzSXnrEg6LZxDI0mS\nes87NJIkqfcMNJIkqfcMNJIkqfcMNJIkqfcMNJIkqfcMNJIkqff+C8iEHEndv5rjAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f39661cea10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,9))\n",
    "plt.plot(1-np.array(train_accs), label='Training Error')\n",
    "plt.plot(1-np.array(valid_accs), label='Validation Error')\n",
    "plt.legend(fontsize=20)\n",
    "plt.xlabel('Epoch', fontsize=20)\n",
    "plt.ylabel('Error', fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAOJCAYAAAA5pdrkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYHFW9P/7PIQFkU4EgiChcBYmyRVmuLLJ4QUCRRVBE\nkEVFQQW+oKDiAoigCKggcv2pFwRBicgiKFeMLMriihfcCIjsmxIlQBKBLPX7ozptVzE9qUm6Z85k\nXq/nyTOn+pyuOjPp7nefc7qqU1EUAQDkZ4mR7gAAMDAhDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtI9\nllKal1K6rgf7uT6lNK8XfRricddq/Q7nDvexYTRIKb0xpXRzSml667ly2Uj3abillLZt/e7HjXRf\nFndjNqRTSpuklM5NKd2dUpqVUnoipfT7lNIXUkqrL+Lue3HyedGj/SzK8YckpbRXSunqlNK0lNK/\nUkr3pZQuTyn9Z5f2u7TejDyRUnoqpfTLlNL+i971yjHWTil9I6V0T6tPj6WUfpFSOqpL+zVSSuek\nlB5OKT3dut+XUkov7GW/FmetF++h/DtgpPvcVEpprYj4QUSsGRHfjIjjI+K7I9ejEdfodSKltE5K\n6aMppWtTSg+klJ5JKT3aen3YdgH3PSCl9OvWa8T0lNJ1KaU3L2rHU0rLppT2TSl9J6U0NaU0I6X0\nZErpNymlo1JKSw5y31enlL6XUvp763Vlakrp+JTS8xa1X3Xje73D0SCldEpEHB0RsyNiSkRMjoil\nImLLiPhIRHwgpXRAURSXLMTuJ0bErB50c/+IWKYH++m7lNL4iDgvIvaJiDujfNF6IiJeHBGvi4jX\nRsSvavf5UEScGRHTIuL8iHg2It4WEd9KKW1QFMXRPejXWyPiOxHxTET8MCLuiYgXRPl/tEdEfLHW\n/hURcXNErBIRl0fE1Ij4z4g4IiJ2SiltWRTFPxe1X2PACVF98U4R8f+i/Nt/OSKm19r/3zD1qxe2\nj4ilI+LDRVFcNNKdGUVOjIi3R8Sfonwu/jPK5+GuEbFrSumIoii+Ur9TSum0iDgqIh6IiK9H+bd/\nR0RcmVI6rCiKry5Cn7aOiG9HxD8i4rqIuDQiVmr16bSIeGtK6b+Konim1qf/jIhrI2JcRHy/1bf/\niohPR8R/te7z7CL0q6ooijH1r/WHnBcRf42IVw1Q/9YoQ3Z2RGw70v0dgb/PWq2/zzlDuM9Jrft8\npkv9+AGO8XREPBYRL+u4/YUR8ZfWvl63iL/H+q1j/DYiXjRA/bgBbru6dewP1m4/vXX7f4/0/89o\n/RcR90bE3M7/79H4r+P1Y5uR7ssI/x22bf0dPt2w/QERsdEAt28d5ZvopyNitVrdFq1j3BkRL+i4\nfc0o39z/KyLWXITfYaMoBxb116flW68b8yLiqFrduIj4c+uxvEvH7SkiLm7d56M9/VuP9H/2MD+w\n1mqF79MRsd4g7d7f+mPfHhGp4/YDW7cfEBE7RcT1UY4Y53W0mRcR1w2wzxdHxLkR8ffWm4D/i3K0\nPP/Bflyt/fWd+23d1m4bEZMi4kdRjkpmttpvPsBxV2+9sNwUEY+2nhAPRcSFMfCblLViCCEdEau1\n9nnTEP4fPjPQ79yqO6hV962O297auu0XAzyh1m/9PR+KiAkdt18REXMiYt2GfXpF6xh/HaBu+YiY\nERFPRcSyI/04Ho3/oktIz3+cR8SSrcfpHa3n57mt+udHOet1bUQ82Hqs/T3KKecB38jNfw5GxMpR\njr4eae3zjxFxYJf7HBDlLMpjUb743x8RP46It7fq5z/3Bvq3dcd+1olyZuihjufaeRGx9gDHPL51\n/20i4p1RzjbNiIh7BqjfJ8rgmBkRD0f5xnGpVrvtI+JnEfFkRDwe5ehwpS6/5xoRcVZE3N36m0xr\n/S036dJ+1Yj4n4j4Wwz8utUopBfw2PhJa19vrd1+fuv2Awa4zwmtuuM7bjuyddv3B2i/fevxd1tE\nPK9Bn/Zp7esHtdvfEN1f4/+jVXdPL587Y226+6BoTVEURfGnQdp9M8ogXDfKJ8j1tfq9ogzpqyLi\n7Cjf2XWqrNOklF4UZcC8LMon081RhvbZUU63P+c+g9wWEbFJRBzT2s/XW8ffMyKuSSlNKorizo62\nW0fER6N8kftdlC8Cr2z9Dru2pnB/3+U4TewV5QvsRSmlZSLizRGxdpSBdkOXfb+h9fPHA9T9b+vn\ndvNvKIri0pTSVyPig1GO2j8aUa4pRcT3Wsd/Z1EU01q3P7/Vj1uLorgjpbRZRGwV/34X/JOiKGbX\njjv/eD+pd6goihkppZsiYocop++v7fK3YOFdGuXj+qpW+e+t218dEZ+N8nlzZZQhtGaUU5I7p5Te\nUhTF1QPs74VRvjF9JsrHyNJRTreek1KaVxTF+fMbppROjoiPRRlcF0X5xnv1iNg0ysf396JcKjkh\nynDaJiK+FeUbj5j/M6W0aUT8NMo3dT+I8rH2qojYLyJ2SyltXxTFbwfo64ejfGxdERHXRLks0Omw\niNg5Ii6L8s3HjlEG0ioppSsi4oIop5C/FuWS3b5RvkF5U+dOUkqvjfLxvWKUz73vR7m0s3tE3JhS\n2qMoiv/taD8hyteY/4iIGyLixtbf5Wvx79etXphd+znfG6J8Dez2OvGpKJ+3x0dEFEXxpZTSdlFO\nUx9aFMV/t36P1aL8G82K8k3X0w36NKf2s7NPMVCfiqK4J6X0l4hYO6X08qIo7m5wnAXrZeLn/i/K\nJ8C8iHhPg7YXtNp+ouO2A1u3zYmIN3a537yIuLZ22/+0bv9c7fYNo3w3+5x3pFG+MZhbu23b+Pe7\n9/1rde9r3f7V2u2rRMRyA/RzwyiD9Kra7WvF0EbS57Xafywi7ovnjjIujohlavd5LMp3tSt22eeM\nVv3zOm5bKiJuad2+Y+u2c2PgWYjtWrdfGuULbL1P90Zt5BARp7bqjuzSp7Na9e8f6cfxaPwXCx5J\n3xoDjP6iHEkPdPtLohyl/nmAuvn/z1+P6kzYq6IMgj/V2v8jypHzc0ZYEbFybfv4qI2eW7enKGfe\n5kbEPrW6t8fAM3Pz9/VUDDwVPL/+8eiYEWo9F/7YOtY/I+L1tX7MH5lu1HH7+Ii4K8qgen3tOC+O\ncpbi4WiNzlu3f721n9Nr7TeO8jMkizySjvIN19Otv0HnlPZyrf0/0eV+E1r1j9RuX6n1fzkryte4\nJaJ84zTgiHyQfv1v6z4H126fP6W9R5f7/bBVv2Ovnjtj7dPdL279fKBB2wdr9+n0g6IonjPiGkhK\naakop06mRzkiaCvKUeb5A91vAW4sOkYCLedE+aTdtHaMx4qimFnfQevY10XEdimlcQvRh/le1Pp5\nYpQjkddE+QR7XZTTc3tGOWPQaf5I4Yku+3yi1i6K8oMYe0c53Xd+SukjUU5R/izK6fOB+vSWKAN7\nnyhHD2tFGcYvi4irUkorL0SffMq7Pz5VDPChvKIonuxy+0MRcUlETEwprTHA/mZGuZ5YdNzn9ihH\nhhNbszDtqijD+zmnPBZF8Y+G/d8iypm3XxRFUfm0d1EU34tyFLpulDM6dV8viuK2QfZ9ZlEUd3Ts\n79koP+yaIuKKoihu6KgrohxgRJQhNd+bI+LlEfGVzvat+zwS5fNitSg/ABWtTzbvG+UU+vG19rdE\nuVy2SFJKS7f2s1SU09adz72Fej62Hiv7RDm7NjnK19w3RMQFRVGc17BfH4pytuL/onxd7TTsrxNj\nLaR75ddDaLtuRDwvIn4/UFhGOSU3VM+ZMiuKYk6U60Yr1utSSm9OKV2ZUnokpfTs/FNfImKXKJ8g\nExaiD/PNfwz9IyLeUhTFbUVR/Ksoil9HOSU5IyLe1YPT2qIoirsi4pAoZwe+EOWI/J2dL8S1Po2L\n8kNgk4uieKIoivuLovholCPsCRFx8KL2iZ4oYpDnVEppy9bpLg+0Toub//j9UKvJSwa421+Kopgx\nwO0PRBlunc+TC6Oc0v1zSunklNKOKaX6lPOCvLb1s9tSyHWtn5MGqFvQ68lAU+SPtH7eMkDdw62f\nnW9eNm/9XKt1qlDlX0Rs1qp/VevnxCjPLrm1KIqnBjjGzxbQ50G1BgbfjvLNzUVFUZy+KPvrVBTF\nTfHv5cqPRfnBs0Ma9uutUZ6B8EhE7FkUxdxe9WthjbU16UejfPC9rEHbl7Z+PjxA3aNDOOb8J/vf\nutR3u30w9VNY5psTZTC1pZSOiIgvRTktNiX+PRVURHka0kZRrtctrPl9uab+olgUxaMppV9H+U52\nkyjX3CLKd5srRfm3eXyAfQ72bnVKlFNjK0TExa1RQN38fc6Lcm2w7rIoP4zWOevwnNF7lz51+9uz\nCIqiGPB5kFLaI8q101lR/t//NcpR8rwoZ0m2iYEfv4M9RyKqz5Mjo5wFOijKF/WPRcSclNJVUZ5q\n9dcGv8L8x8dAj8fO2wcaYS3o9WSg58GcBnWd5/nOnzV62yDHKaKcBYtY8OvWUF4DK1oBfUGU6/2T\no1yzr1vU5+NlUc7upYj4ZlEUCzwtNqW0e5SfSXg0IrYriuLePvRryMZaSN8Q5bru9lF+OGxArQfR\ntq3NgUa63T7QNZAnWz9X7VLf7fZF1jp/+fgoXyBeW38hTClt2YPDTG397PagnB+YnSf53xH/nh78\nZa1PL46IZSPigaL2AY+UUopyeWCFKD+V+v6U0uT69F1r/xERTxe1cxxrfe08D33+77Ful99jndbP\nO7vU0x8nRrlmuUnnlG9ERErpJVGG9CIpimJeRJwREWeklFaJckr6HVEG2noppfWKBZ/3Ov/Fe7Uu\n9S+utat0YYhdXhjzj7trURQ/HEL7bq9P3X7PQbWm0S+MMqAvjPKzNc/5/YuimJlSejgiXpxSWq0o\nivqbgq7Px9YFRb4b5d91ekQcl1K6oqh+oLZ+n7dFeU2FhyPiDYO8MWvyOlEM1K+FNdamu78V5brt\nHimlVw/S7t1RPqmmFkWxSNM6UX5Y5OmI2DCltPwA9QOtUfXKhCjf2d08QEAvH+UU3aK+QPy09XOD\nLvXrtX7e03HbNa2fOw3QfufWz4GmDY+Ocq3ogihH57Mj4jsppZU6GxVFcU+UI6NlUkovH2A/6w/Q\np/nTkTu03gy0pZRWiPJTszOj9qaCvls7yg+H1QN6iejDc6f1GY7LiqLYO8rHxCvi34/hwfyu9XO7\nLvXb1doNt1+0fm7dsP3tUZ6KNql1tkTdtkPtQOvzORdHGdDnFUXxroECusM1UY6Eh/o68cUo1+NP\njvLN1rIRMbl1/IH6tW+Uof5glOe/DzZzMv94z+lT67VmnYi4r/Ua1BNjKqRbf7iTo5wGuiKl9Kp6\nm9aUxxlRThkd2oNjzo5yCuWFEfHJ2rE2ivKcw36Zf072Jiml+dNY89/NnhH/ngJbaK1R7K0RsVXr\nb9eWUjo4yuWFv0R1Xe3cKE+N+VBKac2O9itGxLFRvnH4Wm1fr4vy9Ku/RMShRVH8McppypdE+Qnz\nurOifIKf0vnBuNaHjI5sHaN9xaiiPF3iJ1GuTX6wtq8Tonyif7soin8N8ueg9+6JiFe2Zlgioj2j\ncnyU66eL9CYzpbTUQDNKrefISq39L3CqtLUOekeUz4M9a/vaK8o3FHcURXHjovR3EfwgyqWCD6aU\ndh6oQUpp89ZplPM/43JBlJ+uP77WbpMoP1TWWOtDYpdF+TmVb0Y5EFqQ+a8Bn0gdl+VN5aVZPxit\n8+lrx9kzyvXnG6M862NKlJ9f2SjKZb96vw6Icnbu3ig/sX/vAvr0syjfwGydUnpLx36WiIhTav3u\nibE23R1RPuCWi/JSc7ellK6O8nzGJaOcgt0syiflPj0YRc/3sShHfse0Lin3iyhH6m+L8oIku8cA\nnyyNMmQWWlEU81JKZ7aO/4fWOZVLRfmu/oXR+nT3ohyjZf6nrC9JKV0ZZZCuF+W7zRlRnvrQ+Snb\ne1NKR0d5WdDfppQmRzkq3ivK0D2tKIr2ZURbT9DvRvnG6R3zP4BXFMX/l1L6r4jYK6V0VFEUnZf5\n/Err+HtGxK0ppWujnCbfPcrZhS8OME3+gSg//Xtma7/zLwu6bZQvwJ9YtD/TmNft8TzY4/xLUb7o\n/V9K6dIoHydbRhnQV0b5Cf5FsWxE3JBSuivKUe59US7N7BDlG8wf1EfxgzggWpcZTin9IMrHzLpR\nPuaejP6+IR9UURRzWh+KujoifpRSujnKC3vMivLzN5tG+QZ1tShH0BHlG+b/ioj/1wrmm6J83Xp7\nlK9buw6hC1+LcvQ7Lcop5eNqE1YR5QVC2q+5RVH8IqX0xShfq3+fUrokytevvaN8/TqsKIr757dv\nhfc3o/z8TecHSj8Z5QzCoSmla4qiuLTVfrsoP72dojwV8D0D9OnxoijO6OjTvJTSQVGOqL+fUuq8\nLOjGUb45eM6bgUWysOdujfZ/UT4ovxXltOisKJ9Ev4/yXdfqXe5zQJTT5fsPst/nnCfdun311vHm\nj25/FxHvijJE5kXE4bX210X386QHPDcxylHH3bXbxkU5cvxT67gPRznyfGmU70Ir567GQlwWtON+\n50Z57uqzrZ/nR8Q6g9xnlyifHE9G+WGwX0XEuwZod0mrn0cMUPf8KEcI89ctO+uWjPJa7L9v/e5P\nRMTPI2LvQfq0RpRP3IejHO3fE+X02Qua/B386/p3vaf+WGvd/pzH+QD3PSDK02FmtJ4/l0T5JvC4\n1j7r5ywP+Bxs1VUe81EOVI6O8iIq90UZUH+L8s3a++K5V7gb8Jgd9a9sPe4fXtDzoMG+utbHIK9F\ng71ORHlmxOci4g9RLt88FeWbie9FedWzcbX286841vm6tX+UnwUYymVBr2v1d24MfNW2ud321fpd\nf936/3+ita831dqMj3Ipam5E7D7APl4WZXj/o+P//oCOY3e7mtzdXfr0qtbf7LEoX3umtv6/lu71\ncye1DsgISSmdFBEfj/Lk915exQeAUU5ID5OU0upFUTxcu22DKN+xPx0RLyl6+c0pAIx6Y3FNeqT8\nNpXXdf1TlNNM60R5FaCI8tJzAhqACiPpYZJS+nSUHyBZK8oPMD0e5RrKaUVR/HwEuwZApvoa0iml\nUxbcClgYRXmJ075rnUq39nAcC8agfxVFcXy3yn6HtGE69ElRFIt0il5TKaUpUV6lD+i96UVRPOc7\nF+YbUxczAYDRREgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgD\nQKaENABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgDQKaE\nNABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkavxIdwCA3ho3blyjdnPnzu1zT1hURtIAkCkh\nDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZcllQYExaYonmY5ShtJ0wYULj\ntgcffHDjthtuuGHjtssuu2xP20VE7L777o3bPvHEE43bMjgjaQDIlJAGgEwJaQDIlJAGgEwJaQDI\nlJAGgEwJaQDIlJAGgEwJaQDIlJAGgEy5LCgwJu24446N25577rmN286dO7dx26WXXrpx22uvvbZx\n26eeeqpRu6H8Dc4///zGbXfbbbfGbRmckTQAZEpIA0CmhDQAZEpIA0CmhDQAZEpIA0CmsjkF613v\nele7/O1vf3sEewIAeTCSBoBMCWkAyNSITXenlCrb3/jGN9rlV7/61ZW6j3/848PSJ2DsmDFjRuO2\nTa/gFTG0K3N99rOfbdx20qRJPe/DUK6O9tWvfrVxW3rHSBoAMiWkASBTQhoAMpXNKVidjjzyyMr2\nH/7wh3b5O9/5znB3BwBGhJE0AGRKSANApoQ0AGQqmzXp22+/vV3eaKONKnWd5wdakx476ufS33DD\nDV3r9t5778r2gw8+2L+OAQwTI2kAyJSQBoBMpaIo+rfzlBrvvD592U0/+0te6o+JZ599tl1eYonq\n+8ubbrqpsr311lv3r2OZKIqi2ZNmES2xxBJTImL7Jm0X1+dn/fE2mAkTJjRue8ghhzRue8QRRzRu\n+8IXvrBRuyuuuKLxPt/+9rc3bjt79uzGbYnpRVGs2K3SSBoAMiWkASBTQhoAMpXNKViL61oWvdO5\nLlhfr95iiy0q23vttVe7/Mtf/rJS5/QsYLQwkgaATAlpAMiUkAaATGWzJg0LMm/evHa5ft5q/TMN\nF110Ubt88803V+rGwjnUwOLBSBoAMiWkASBTprvJVn0Ke6mllhqhnoxtTo+M2GeffRq3Pf/88xu3\nbXo55KFq+n82a9asxvt0qc+RYSQNAJkS0gCQKSENAJmyJs2osbisjR5++OGV7fpXAG611VbD2R0g\nY0bSAJApIQ0AmTLdDcNg/Ph/P9V23333EewJMJoYSQNApoQ0AGRKSANApqxJwzCYMGFCu7zttttW\n6urf0kV+fv3rXzdu+89//rNx28svv7xx2y984QuN26622mqN2p144ol9Of4xxxzTuC2DM5IGgEwJ\naQDIlJAGgExZk4ZhMNhXHf785z8fxp4Ao4mRNABkSkgDQKZMd0MfLLvsspXt17/+9e3y448/Xqn7\n/Oc/Pyx9AkYfI2kAyJSQBoBMCWkAyFQqiqJ/O0+pfzuHjNVPubrgggva5QsvvLBSt//++y/UMYqi\nSAt1xyFKKU2JiO2H41iLg6WXXrpx22effbZx26G8VqfU7KGxyy67NN7nGWec0bjty1/+8sZtielF\nUazYrdJIGgAyJaQBIFNOwYI++O53v1vZvuiii0aoJ8BoZiQNAJkS0gCQKSENAJmyJg3DoJ+nOgKL\nLyNpAMiUkAaATAlpAMiUNWkge8stt1zjtoccckijdg8++GDjff7hD39o3PbPf/5z47bjxo1r3Hbu\n3LmN2za9LOihhx7aeJ+zZ89u3HaZZZZp3PZf//pX47ZjkZE0AGRKSANApoQ0AGRKSANApoQ0AGRK\nSANApoQ0AGRKSANApoQ0AGTKFcd6oPNqSBMmTKjU3Xfffe3yKqusUqlbdtllK9ud9505c2alburU\nqV3389hjjw2xx9DcGmus0bjts88+27jtZptt1rjtO9/5zsZt3/GOdzRu29RQvsXsrrvuatx26aWX\nbtz2uuuua9x2hx12aNRu1VVXbbzPz3zmM43buopY7xhJA0CmhDQAZEpIA0CmrEn3wB577NEun3rq\nqZW6Bx54oF2+4YYbKnWbbLJJZXuLLbZol+trOp1r0p3liOeug994443t8sknnzxo3wHIl5E0AGRK\nSANApoQ0AGRqsVuTnjhxYrtcX7vtlZRSZXurrbZql1/0ohdV6jrPQ9x4440H3U/nuZjLL798pa7z\nvvW17Po5nJ1tL7jggkrd/fffHwCMDkbSAJApIQ0AmVrsprv7NcXdqT69fMcdd3StG+x+9VOybr/9\n9nb5zjvv7Hrf17/+9ZW63XbbrbK98sort8vvfe97K3Wf/vSnu/YPBrLaaqs1bnvzzTc3bvvII480\nbtuPS5MO5ZKcQ7ks6DrrrNO47VDsv//+Pd/nlVde2bjtaaed1vPjs2BG0gCQKSENAJkS0gCQqcVu\nTXokfPnLX26XO7+aMqJ6yc7LLrusUjdt2rTGx+j8esr6ZUB33333ynbnqV3ve9/7KnXWpAFGDyNp\nAMiUkAaATAlpAMiUNeke6DyH8tJLL+3JPj/xiU9Utt/znve0y2uuuWbX49e3P/e5z/WkPwAMPyNp\nAMiUkAaATKWhXO5uyDtPqX87z9Qee+xR2R7sW6cOPvjgynbnN3jVL/3ZabBvz4qI+MlPftIu77zz\nzt07y6hWFEVacKtFl1KaEhHbN2m70korNd7v448/vrBdGlTn82gwxx9/fON9br311o3bzpkzp3Hb\n+mmZvWp77733Nmr30EMPNd5n08utMmTTi6JYsVulkTQAZEpIA0CmhDQAZMopWD3QuUZ87LHHVupe\n85rXtMv1teMllqi+RxrK11x2qp/29YEPfKB7ZwEYNYykASBTQhoAMmW6u886p7TrU9b1U6ma1k2d\nOrWyfeCBB1a2Z86cOYQeApArI2kAyJSQBoBMCWkAyJTLgvZA5/rxr3/960rda1/72nZ5QWvSnfUL\nuvRnp5NPPrmy/elPf3oBPWa4ve51r2uXf/nLX/ZknzleFnQ0GexzH3XLLLNM47bPPPNM47bz5s1r\n3Lafr9WMKJcFBYDRSEgDQKaENABkypp0j9XXuTq/jvL222+v1N14442N9/uJT3yiXf7MZz5Tqfvz\nn/9c2V5//fUb75fe6fy/nzx5cqVuzz33bJfHjRvXk+NZk1401qTJhDVpABiNhDQAZMp09yjROTV3\n4oknVuo++tGPVraPO+64drl+ehb9s/nmm7fLN998c6Wu83lW//azhWW6e9GY7iYTprsBYDQS0gCQ\nKSENAJkadWvSb3nLWyrbV1555VD60y7/6le/qtRtvPHGle3O06U+97nPVeouu+yyyvasWbMa96EX\n1lprrcp2/Xe577772uXNNttsOLo0Jnz84x+vbNdPhfvXv/7VLq+wwgqVuvvvv79dXnPNNXvSH2vS\nsFiwJg0Ao5GQBoBMjR/pDjTROU19ySWXVOr+8Y9/dL3f1772ta51m2666aDHXG+99drl8847r1JX\nP+Wp81unLr/88kH3u7A6/wbvec97KnUrr7xyZbtzupveqZ+ys6DtTvUlCYAmjKQBIFNCGgAyJaQB\nIFOj4hSszrW+2bNnL9T9IqqX1RvKNxHV/0aD/c0uvfTSynbnGnX9W6+Gsnb8qle9ql2ur8uvu+66\nle2rr766XX7Tm97U+Bg8V+dj6De/+U2lbtKkSZXtzst91h97TsECunAKFgCMRkIaADIlpAEgU6Pi\nPOnONeCNNtqoUrf++utXtjvPb95tt9261i3s8Qfa7lx/3GOPPSp1nduPPfZYpe4rX/lKZbvzUqT1\nS1BOnDixXV5uueUqdfWvu+vXudpj3QMPPFDZrq9JD3ae9Etf+tK+9AlYvBlJA0CmhDQAZGpUnII1\nxGM2arfMPapaAAAfcUlEQVTBBhtUtm+77baubYcy3T2UusGO03k6z4Lqrr/++sr2dtttN+hxWDib\nb755ZfunP/1pZfuLX/xiu3zGGWdU6jovX9ur55xTsGCx4BQsABiNhDQAZEpIA0CmFrs16abqa9f7\n7bdfZfvUU09tl1dZZZVKXa/WpAe7bOlgdfXLix511FGV7d/97ndB/w32+Yd+Pq86jmFNGkY/a9IA\nMBoJaQDI1Jid7q6rT112XuHrpJNOqtTVr2Q2HNPdnd+u9clPfrJSN3Xq1GDsMd0NiwXT3QAwGglp\nAMiUkAaATFmTbqD+rVOd69XDpXPdeebMmcN+fPJjTRoWC9akAWA0EtIAkCkhDQCZGj/SHRgN6mvA\nt9xyywj1BICxxEgaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU0Ia\nADIlpAEgU0IaADKViqIY6T4AAAMwkgaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiU\nkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaA\nTAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlp\nAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiU\nkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaA\nTAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlp\nAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiU\nkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaA\nTAlpAMiUkAaATAlpAMiUkAaATAlpAMiUkAaATI3v585TSkU/9w9jWVEUaTiOk1KaEhHbD8exYAya\nXhTFit0qjaQBIFNCGgAyJaQBIFNCGgAyJaQBIFNCGgAyJaQBIFNCGgAyJaQBIFNCGgAy1dfLggLA\nWDR+fG/i1UgaADIlpAEgU6a7GVEpVb/I6ZRTTmmXjzzyyK73u+CCCyrbBx10UG87BpABI2kAyJSQ\nBoBMCWkAyJQ1abJywAEHtMv19eqiKNrl888/f9j6BDBSjKQBIFNCGgAyZbqbrEyYMKFd7pzejoi4\n77772uWpU6cOW58Ahmr55ZfvyX6MpAEgU0IaADIlpAEgU9akycoSS/z7feO8efMqdd/97nfb5Uce\neWTY+gQwUoykASBTQhoAMiWkASBT1qTJSuc6dH1N+rHHHhvu7gCMKCNpAMiUkAaATJnuJlszZsyo\nbN94440j1BOAoZk+fXpP9mMkDQCZEtIAkCkhDQCZsiZNVjq/nvKEE06o1P3ud78b7u4AjCgjaQDI\nlJAGgEwJaQDIlDVpsjVt2rSR7gLAiDKSBoBMCWkAyJTpbkZU5ylXERFLLrlk1zqgt9Zaa63GbffY\nY4/GbbfbbrvGbevfdjeYc845p3HbK664onHbnBlJA0CmhDQAZEpIA0CmrEmTFevQAP9mJA0AmRLS\nAJApIQ0AmRLSAJApIQ0AmRLSAJApp2CRrX322aey/e1vf7td3nHHHSt111xzzbD0CXpp/PjmL8E7\n7bRT47aHHXZYo3ZveMMbGu9ziSWaj+lSSo3bzpkzp3Hb3/zmN43buiwoANBXQhoAMiWkASBT1qTJ\n1sUXX1zZPvLII9vlrbfeulJnTRpYHBlJA0CmhDQAZMp09xizww47tMubbLJJpW6DDTboer+77767\nsv3Vr361sv3II4/0oHdV9VMzLr/88nb5jW98Y6Vu+eWXr2zPmDGj5/0BGG5G0gCQKSENAJkS0gCQ\nKWvSo9BGG21U2f7Qhz5U2d50003b5XXWWadSt/TSS7fL9Uv3DXYpv6IoKtt77bVXZXvixImD9Lg3\nZs+e3S6//vWvr9TV/waf//zn+94fxo7llluucdu3vvWtjdt++MMfbtx2/fXXb9z2ySefbNTu6aef\nbrzPofwNmh4/IuKUU07pS9t+WH311Ru3ffTRR3tyTCNpAMiUkAaATJnuHkGdU88REfvtt1+7vOee\ne1bqNttss3a5frrRUkst1YfeVdWnwuvT6MOt3p9x48aNUE8A+sdIGgAyJaQBIFNCGgAyZU26zzrX\nmSdNmlSp22mnnSrbr3rVq9rlwU6HWpD77ruvXa6ve6+22moLtc/66QSHH374Qu2nV+qnhAEsjoyk\nASBTQhoAMiWkASBT1qR7rH7ZuJNOOqldfulLX9p4P3/9618r29/61rfa5cmTJw9632nTprXLp512\nWqXu3e9+d9f7dV52MyLi7LPPbpevueaaSt0Pf/jDQfvQDz//+c/b5fqa/ZZbbjnc3SFTyy67bKN2\n22yzTeN9nnDCCY3bbrjhho3bTpkypXHbD37wg43b/u1vf2vUbijP45e97GWN2x5//PGN2375y19u\n3HakPfzww8N+TCNpAMiUkAaATJnuXgj1qdbPfe5z7fJBBx1UqVtllVW67mfq1KmV7Tlz5rTL9enb\np556qnF/DjnkkHb5Xe96V9f7Pf7445Xtr3/965XtY489tl3u1SlPa6+9dmX7kUceqWzPnDmz633/\n/ve/d62bPn36onUMIENG0gCQKSENAJkS0gCQKWvSC6FzzTci4qijjmqXx49v/iedOHFiZftXv/pV\nuzxv3rxK3Ute8pJ2eYsttqjU7bXXXpXt3XffvV1ecsklK3UPPPBAu1z/PX784x9Xtvtx6c277rpr\noe97zz33tMu+mhIYC4ykASBTQhoAMpX6+W1CKaXF4quK6qc4zZgxo7K9zDLL9GS/neqnZ73yla9c\nqP3cdNNNle3OKe4//elPC+oiGSuKYuG/Km0IUkpTImL74ThWL2y33XaN2g3laltPPvlk47aXXXZZ\n47Yf/vCHG7fdaKONGrc9/fTTG7Vbb731Gu/ziCOOaNz2/PPPb9zWN9rF9KIoVuxWaSQNAJkS0gCQ\nKSENAJlyClYDL3/5yyvbQznNamGtu+66Xevqa9D1NZ1LL720XT766KMrdZ2nMQGQNyNpAMiUkAaA\nTAlpAMiUNekGdtppp8p2/VKbTdUvu7nzzjt3bXvrrbdWtidNmtQu19egzzzzzMr2Rz/60Xb5mWee\nGXI/AciDkTQAZEpIA0CmXBa0gfopTy9+8Ysr28stt1y7PJRvearv97e//W27vNpqq1XqVl111Xb5\ngAMOqNRdeOGFlW2X2RsbxtJlQQ899NDGbU855ZRG7ZZffvnG+xzs0rt1jzzySOO2Q1mOWnPNNRu3\nnTVrVqN2Q7nU5znnnNO4rdegIXFZUAAYjYQ0AGRKSANAppyC1UB9feXhhx/uyX7f9KY3VbZf85rX\ntMuzZ8+u1B144IHt8gUXXNCT4wOQNyNpAMiUkAaATAlpAMhUNmvSnZfIvPLKK7u2q68P77bbbpXt\nq666qrcd66H999+/sl0/n/Opp55ql9/2trdV6q6++ur+dQyALBlJA0CmhDQAZGrEprvHj68e+g1v\neEO7vMQS3d87fOUrX6ls5zy9HRGx1VZbtcunnXZapW6FFVaobHeeknXdddf1t2MwwoZyqc36c2cw\nyyyzzMJ0p2fqlw0ezIMPPti47VAutfnxj3+8UTuX+syfkTQAZEpIA0CmhDQAZGrE1qR32GGHyvZR\nRx3VLg+29vHkk0/2rU+9sMUWW1S2L7/88na58ystIyI+8IEPVLbH4jr0McccU9k++eSTu7btXD97\n3/ve17c+AeTCSBoAMiWkASBTwzbdXT/d4rDDDmt833vvvbddPuOMM3rVpZ5Zc8012+Uf/ehHlbrn\nP//57fIHP/jBSt25557b346NAmeddVbXus9+9rOV7Xe/+93tsuluYCwwkgaATAlpAMiUkAaATA3b\nmvR//ud/VrYnTZrUtW39FKyddtqpXZ42bVpvO7YQ6pcdPP3009vlF7zgBZW6O+64o13uPB2L0qxZ\nsyrbp556art86623VuqOPfbYYekTw2OttdZq3HbevHmN286dO7dRu+9973uN93n22Wc3brv22ms3\nbtv52rEgQ3n81y+fzOhlJA0AmRLSAJApIQ0AmerrmnTnudH1yz2uuuqqXe9XX3eur1sOt6WXXrqy\nfeGFF1a2d99993Z59uzZlbrOr5989NFH+9C7xUvn5xF+8pOfVOr8/YCxxkgaADIlpAEgU8N2CtY2\n22wzaH3nNOd73/veSt1DDz3Ulz4NpnOqfvLkyZW6XXfdtbLd2fcjjjiiUnfPPff0oXdjwyte8YrK\n9g033NAu1091A1gcGUkDQKaENABkSkgDQKaGbU16QTrXgA866KBK3fXXX98uz5gxo1I3ceLEyvYG\nG2zQLl988cU96Vv9FLCLLrqosn3LLbe0y9/61rd6ckyql4ONiFhuueVGqCf0w1A+r7Haaqv1/PhD\nObVzjTXWaNz2lFNOadz2O9/5TuO2Q7mEKIsPI2kAyJSQBoBM9XW6u/PUpPHjF/5Q9W/F6jR16tRB\nt3txzH333bdxWxZN57LH+uuvP4I9ARh5RtIAkCkhDQCZEtIAkKlhOwVrNK/bjua+j2Z77rnnSHcB\nYEQZSQNApoQ0AGRKSANAprK5LChEVNf/V1111RHsCTmZOXNmz/e55JJLNm77iU98onHbOXPmNG57\n7LHH9mW/LD6MpAEgU0IaADJluptsOfUNGOuMpAEgU0IaADIlpAEgU0IaADIlpAEgU0IaADLlFCxg\nsZJSatTuQx/6UON97rrrro3bbr/99o3bzpgxo3FbxiYjaQDIlJAGgEwJaQDIlJAGgEwJaQDIlJAG\ngEwJaQDIlJAGgEwJaQDIlJAGgEy5LCiwWFl11VUbtTv00EMb7/Oiiy5q3PaOO+5o3BYWxEgaADIl\npAEgU0IaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgUy4LCmRvxRVXbNx28uTJjdot\nu+yyjfd59tlnN247d+7cxm1hQYykASBTQhoAMiWkASBTQhoAMiWkASBTQhoAMiWkASBTQhoAMiWk\nASBTQhoAMuWyoED2llpqqcZtf/aznzVqd/TRRzfe51133dW4LfSSkTQAZEpIA0CmhDQAZEpIA0Cm\nhDQAZEpIA0CmhDQAZEpIA0CmhDQAZEpIA0CmUlEU/dt5Sv3bOYxxRVGk4TjOcsstNyUitm/Sdtas\nWX3uzYKl1OzP0s/XPhiC6UVRrNit0kgaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU0IaADIlpAEg\nU0IaADIlpAEgU+NHugNA3nK41OdQuNwnixMjaQDIlJAGgEwJaQDIlJAGgEwJaQDIlJAGgEwJaQDI\nlJAGgEwJaQDIVHJ1HgDIk5E0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRK\nSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANA\npoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0\nAGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRK\nSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANA\npoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0\nAGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRK\nSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANA\npoQ0AGRKSANApoQ0AGRKSANApoQ0AGRqfD93nlK6u5/7h7GsKIqXD8dxUkrfjogth+NYMAY9WRTF\npG6VfQ3piPiPPu8f6L/VwnMZ+mX6YJWmuwEgU0IaADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU0Ia\nADIlpAEgU0IaADIlpAEgU0IaADIlpAEgU/3+FizGoD322KOyffHFF7fLTz75ZKXuqquuqmyfc845\n7fK1117bh94BjB5G0gCQKSENAJky3U3PTZ9e/Q7zWbNmdW373//935Xtm266qS99AhiNjKQBIFNC\nGgAyJaQBIFPWpOm566+/vrL9qU99ql0+/fTTK3UbbbRRZfuPf/xju/zEE0/0vnNA29prr9247THH\nHNO47QknnNC47UMPPdS47VhkJA0AmRLSAJAp0910NWHChHZ5ypQplbrXvOY1Xe9XFEVlu/OKY29/\n+9srdWeeeWZle++9926Xt9lmm+adBVgMGUkDQKaENABkSkgDQKasSdOWUqpsX3LJJe3y+uuvv9D7\nffjhh9vlPffcs1J3+eWXV7a33HLLhT4OwOLGSBoAMiWkASBTQhoAMmVNmraJEycOut0Ljz76aGV7\nl112qWwfeeSRPT8mMLB99923cdsDDjigcduf/exnjdteeOGFjduORUbSAJApIQ0AmTLdTVvnZUAj\nIlZeeeV2uX6pz16ZNm1aZfuTn/xkX44DMBoZSQNApoQ0AGRKSANApqxJk5V+rX0DjEZG0gCQKSEN\nAJky3Q30zFZbbdW47cEHH9y47bXXXtu47Xnnndeo3SabbNLzfUZE/M///E/jtl/84hcbt+2Hp556\nqnHbcePGNW47e/bshekOAzCSBoBMCWkAyJSQBoBMWZOm7e9//3tl+29/+1u7/KIXvWi4uwMw5hlJ\nA0CmhDQAZEpIA0Cmhm1NOqVU2X7mmWca3/f3v/99u3zFFVc0vl/9EpMnnnhi4/uORXfeeWdle/Lk\nye3yYYcdNtzdARjzjKQBIFNCGgAyNWKnYP385z+vbN9+++3t8umnn16pmzFjRrs8bdq0/nZsDKsv\nD3z4wx8esMzYcuaZZzZu+853vrNx25VWWqlx21VXXbVx26aX8Nx8880b73Pddddt3Paxxx5r3LZf\n6suL3ay33nqN9zmU196rrrqqcVsGZyQNAJkS0gCQKSENAJkatjXp+nrnDjvs0LgtI8P/A8DIMpIG\ngEwJaQDIlJAGgEyN2HnS1jsBYHBG0gCQKSENAJlK/Zx2TimZ04Y+KYqi2bUfF92UiNi+1zv94x//\n2Ljt0Ucf3bjtfffd16jdT3/608b7fPrppxu33XrrrRu3feihhxq3HYoVVlihUbu777678T7//Oc/\nN2672267NW47ffr0xm0XU9OLolixW6WRNABkSkgDQKaENABkSkgDQKaENABkSkgDQKaENABkSkgD\nQKaENABkSkgDQKZG7FuwRpOUqldf3GuvvSrbG2+8cdf7fuxjH+tLn3rhbW97W2X7oosuqmzvuuuu\n7fKPfvSjYekT+RnK5TMPP/zwxm3vv//+xm1nzZrVuO0pp5zSqN3qq6/eeJ+3335747aTJk1q3Lb+\nHBzMtGnTGrfdcsstG7V7wQte0HifSy65ZOO29I6RNABkSkgDQKaENABkasyuSa+4YvWbwebMmVPZ\n3nnnndvlHXfcsVJ34IEHVrY716zrX/052Jp0fa376quvbpfHjRtXqTvmmGPa5VtuuaXrPhek85iD\n/R4RER/5yEfaZWvSAMPPSBoAMiWkASBTi/V0d3369qSTTmqXd9lll0pd/RSPTTfdtOt+emXrrbeu\nbG+77bbt8hJLVN8/7bfffu3yokx3D8Wdd945LMcBYGBG0gCQKSENAJkS0gCQqcV6Tbout0t0vu51\nr6tsd552VT8l7KqrrurJMTsvA7j55ptX6uqnj/3hD3/oyTEZ3d74xjc2blt/DI2ERx99tFG7Bx98\nsPE+V1lllcZtzzvvvMZt66eCDmYon41p2nYol1t94IEHGrcdP35MRUtfGUkDQKaENABkypzEQnj6\n6acr29/85jfb5XPOOafr/bbbbrvK9sknn9y17d13313ZnjJlylC62NX222/fLg/lG3AAGH5G0gCQ\nKSENAJkS0gCQqTG1Jt25BnzsscdW6iZPnlzZnjZtWrv85S9/uVJXX5N++OGH2+X6KSjrrrtuu1w/\nNaN+6c9//vOf7fJ73vOe5/4CC6F+Ksaee+7Zte62226rbH/jG9/oSR8AWDhG0gCQKSENAJkS0gCQ\nqdTPy/illEb+GoEdFvYrJ4fyN6of47jjjmuXP/WpTw3a9he/+EW7vOWWWw6li1296EUvqmzfeuut\n7fKqq65aqfvsZz9b2e7sO/kpiqI/36Fak1KaEhHbL7DhKFP/TMhghnJNgenTpzduu/TSSzdu+4EP\nfKBx21NOOaVRu+9973uN9/m+972vcduZM2c2bktML4qi6/VhjaQBIFNCGgAyNaZOwRqJb+jZYost\nGrddY4012uWjjjqqUnfnnXe2yxtvvHGlrvOypBHVU8LqU2SdU9z1v8cll1zSuK8A9J+RNABkSkgD\nQKaENABkakytSQ+H+jpv56VI66c8vfKVr6xsd65Jf+ELX2h8zMMPP7yyfccdd7TLG220Udf73XPP\nPZXt3//+942PCUD/GUkDQKaENABkakxdcWyk1a8wVp+Kfstb3tIuv/Wtb63Ubbjhho332/T/tN5u\n++2rF5W6/vrrG+2HkeGKY6PfUK6CeNZZZzVu+/73v79Ru/qS22Duvvvuxm0ZElccA4DRSEgDQKaE\nNABkyilYw6i+Btz5jVT17fo3UnXabrvtKts77bRTZfsjH/lI1/vee++97fKPfvSjrnUAjDwjaQDI\nlJAGgEwJaQDIlDXpTA12rvN1111X2f7Sl77U9b71/ey///7t8k033dT4mAAMPyNpAMiUkAaATJnu\nHoXqlwjdYIMNKtvz5s1rl+uXB/ztb3/bLpvehpE12LfU1R1yyCGN21599dWN2v3tb39rvE9GhpE0\nAGRKSANApoQ0AGTKmvQoscwyy7TLZ5555qBt58yZ0y5fc801lbqnn366tx0DoG+MpAEgU0IaADIl\npAEgU9akR4k3v/nN7fJWW201aNtLL720Xfb1kwCjl5E0AGRKSANApkx3Z2qdddapbE+ePLldTilV\n6m677bbK9rvf/e7+dQwY1FJLLdW47eGHH964bf15P5iTTjqpUbuZM2c23icjw0gaADIlpAEgU0Ia\nADJlTTpTO+64Y9e6zq+ijIg47rjjKtsu/QmweDCSBoBMCWkAyJTp7oxMmjSpXT711FMrdXPnzm2X\nr7322krdFVdc0d+OATAijKQBIFNCGgAyJaQBIFPWpEdQ/TJ/Rx99dLtcv7TgQw891C7vvPPO/e0Y\nsNBe+tKXNm677777Nm573nnnNW570003NW5L3oykASBTQhoAMiWkASBT1qQzsuSSS7bLTz75ZKXu\nne98Z7tcFMWw9QmAkWMkDQCZEtIAkCnT3SOoPm299957N24LwOLPSBoAMiWkASBTQhoAMmVNOiPW\nnWH0G8ple59++unGbU888cSF6Q6jnJE0AGRKSANApoQ0AGRKSANApoQ0AGRKSANApoQ0AGRKSANA\npoQ0AGRKSANAplI/L0WZUnKdS+iToijScBwnpTQlIrYfjmONNSk1/y902eDF1vSiKFbsVmkkDQCZ\nEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZGj/SHQAYq1xFjAUxkgaATAlpAMiU\nkAaATC12a9Lvfe972+WXvexllbqzzz67XX700UeHrU8AsDCMpAEgU0IaADIlpAEgU6N+TXrixImV\n7bPOOqtdHj+++uvdfPPN7fKPf/zj/nYMABaRkTQAZEpIA0CmRv1095vf/ObK9pJLLtkuu+QeAKOZ\nkTQAZEpIA0CmhDQAZGrUrUmnlCrbn/zkJ0eoJwDQX0bSAJApIQ0AmRLSAJCpUbcmXff85z+/sn3b\nbbe1y+utt95wdwcAesZIGgAyJaQBIFOjfrp76tSple2tt966Xb7//vuHuzsA0DNG0gCQKSENAJkS\n0gCQqVG3Jl3/+sn111+/a/3KK6886H0BIGdG0gCQKSENAJkaddPddYNNYY/V6e3nPe957fJBBx1U\nqdtvv/3a5S233HLY+gTA0BlJA0CmhDQAZEpIA0CmRv2aNBE77LBDZftrX/tau7zCCitU6q6//vrh\n6BIAPWAkDQCZEtIAkCkhDQCZsiY9Cm266aaV7QsvvLCy/f3vf79dPv744yt1jz32WN/6BUBvGUkD\nQKaENABkKvXz0pkppbF5Xc4+W3755SvbK664YmX7wQcfbJfH6qVRx4KiKNJwHCelNCUith+OY8EY\nNL0oihW7VRpJA0CmhDQAZKrfn+7+XZ/3D/TfXyJipZHuBCymnhqssq9r0gDAwjPdDQCZEtIAkCkh\nDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZEtIAkCkhDQCZ\nEtIAkCkhDQCZEtIAkCkhDQCZEtIAkKn/H7/oRdphKWJdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3965c706d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.figure(figsize=(7,14))\n",
    "for i in range(3):\n",
    "    plt.subplot(321+i*2)\n",
    "    plt.imshow(data['X_test'][i].reshape(DIM, DIM), cmap='gray', interpolation='none')\n",
    "    if i == 0:\n",
    "        plt.title('Original 60x60', fontsize=20)\n",
    "    plt.axis('off')\n",
    "    plt.subplot(322+i*2)\n",
    "    plt.imshow(test_transform[i].reshape(DIM//3, DIM//3), cmap='gray', interpolation='none')\n",
    "    if i == 0:\n",
    "        plt.title('Transformed 20x20', fontsize=20)\n",
    "    plt.axis('off')\n",
    "plt.tight_layout()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# References\n",
    "[1] Jaderberg, Max, et al. \"Spatial Transformer Networks.\" arXiv preprint arXiv:1506.02025 (2015).\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
